{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 线性分类器 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_toy_data(add_outliers=False, add_class=False):\n",
    "    x0 = np.random.normal(size=50).reshape(-1, 2) - 1\n",
    "    x1 = np.random.normal(size=50).reshape(-1, 2) + 1\n",
    "    if add_outliers:\n",
    "        x2 = np.random.normal(size=10).reshape(-1, 2) + np.array([6., 3.])\n",
    "        return np.concatenate([x0, x1, x2]), np.concatenate([np.zeros(25), np.ones(30)]).astype(np.int)\n",
    "    if add_class:\n",
    "        x2 = np.random.normal(size=50).reshape(-1, 2) + 3.\n",
    "        return np.concatenate([x0, x1, x2]), np.concatenate([np.zeros(25), np.ones(25), np.zeros(25)+2]).astype(np.int)\n",
    "    return np.concatenate([x0, x1]), np.concatenate([np.zeros(25), np.ones(25)]).astype(np.int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x20d2e1629b0>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3xW9fn/8dd17zuDHRAZIk4URWpw741aEUfdOFqx/rRqa21VtK2z9mvV1lWlddTWYh217oGV4h5gERWcKAoyAgkryT3P9fvjjkC474Qk95375Ny5no9HHg9yzrnPeR+FKyef8xmiqhhjjPEun9sBjDHG5McKuTHGeJwVcmOM8Tgr5MYY43FWyI0xxuMCbly0X79+OmzYMDcubYwxnjVz5sxlqlq14XZXCvmwYcOYMWOGG5c2xhjPEpH5ubZb04oxxnicFXJjjPE4K+TGGONxVsiNMcbjrJAbY4zHWSHvgjQxA6d2As7SvXBqz0QTs9yOZIzpwlzpfmhapvFX0brzgFhmQ2IpWjsTek9Gwru5ms0Y0zXZE3kXo6uuY20RXyuGrr7ejTjGGA+wQt6FqCqk5+XemfqsuGGMMZ5hhbwLERGQnrl3+voUN4wxxjOskHc15T8Cos23SRTKz3YljjGm67OXnV2MlJ+NOmug4QGQpo1lZyFlp7uayxjTdVkh72JEfEiPi9HK8yBdA/7+iITdjmWM6cKskHdRIhEIDHE7hjHGA6yN3BhjPM4KuTHGeJwVcmOM8Tgr5MYY43FWyI0xxuOskBtjjMdZITfGGI/Lu5CLSERE3hGR90XkIxG5qhDBjDHGtE0hBgTFgQNUdY2IBIHXROQ5VX2rAOc2xhizEXkXclVVYE3Tt8GmL833vMYYY9qmIG3kIuIXkVnAUmCqqr6d45iJIjJDRGbU1NQU4rLGGGMoUCFX1bSq7gQMBnYRkZE5jpmsqtWqWl1VVVWIyxpjjKHAvVZUdQXwX+CwQp7XGGNMywrRa6VKRHo1/TkKHAR8nO95jTHGtE0heq0MBP4qIn4yPxgeVtWnC3BeY0qeqjatx6oQ2Dqz3J8x7VSIXiuzgdEFyGJMt6LJ2WjdT0BXZjZIJfS6HQmNcjeY8Rwb2WmMC9RZg9aeAc4i0IbMl7MErTsDdVa5Hc94jBVyY9wQex7UybHDgdizRY9jvM0KuTFucJaRGRS9AY017TOm7ayQG+OGUDVIKHu7RCFYXfw8xtOskBvjhuDOEBwDRNfbGIXgThDa1a1UxqMK0f3QGNNOIgK970IbHoXGRzIbo8ciZcdbF0TTblbIjSkQVQeSMyG9FIKjkMDgVo8XCSDlJ0L5iUVKaEqVFXKTkyY/RGMvg4SRyOFIYIjbkbo0TS9CayeAUwMIaAqNHo30uNqesE2ns0JumlFVdNU10PgokAD86Jrb0R6/wVd2rNvxuiytOx/SX9NsBufGJyA0GqLHuJbLdA/2stM0l5wBjY8BMcABkkAcVv0GderczdZFaXoRpD4hexr+GLrmbjcimW7GCrlpRhufIVPEN+SH+CvFjuMN2tDyvvT8TNu5MZ3ICrnZgB/I0aYr0rTPZPFvTua3l5w7IfVRMdOYbsgKuWlGokcB4ewdmobwvkXP4wUivqZinmunrXxoOp8VctOMhEZB+ZlkinkIiGS+ev4e8VW6G64Lk/KzyPy32nBHFALbFz2P6V5KpteK0/gMrLkJ0ovAvylU/Axf9Ai3Y3mSr/IiNHo0xP8LEobwIYi/r9uxurboOIi/CIm3MvOlEAbxIb1uJTNVvzGdpyQKudP4FKycxNqXdOlvYOVlOGDFvIMkMAwCZ7gdwzNEAtDrrkyvn8Q74OsNkcMRXy+3o5luoCQKOatvIrunRSzzhG6F3BSJiEBoTObLmCLyfBu5qoLzbe6d6YXFDWNMO6k6aPJTNPWV21GMh3n+iVxEUN/AzEorG/INLH4gY9pI42+iKy9uWiHIQf2DkN53IIHhbkczHuP5J3IAKn5Kdo+BSNN2UyrUWY3GnkdjL6JOK4NwPEDTi9AVP84sIqENQAzS89DaU1FNuh3PeEzeT+QiMgR4ANiEzKiIyar6x3zP2x6+sqNxUFhzCziLwbdJptdK2bhixjCdyGl8GlZeDvLdgCUHev0BCe/nWibVNCQ/zAyWCmzfrt4p2vBYpm9+862gjRB/DSL7FzasKWmFaFpJARer6nsiUgnMFJGpqjqnAOduM1/ZeCgbj6rabHMlRtMLYeVlQLzZ2BqtuwD6T0d8vYufKfFuZqIsEpkNEoFedyKh0W07gbN43WebnTjdNIOiMW2Xd9OKqi5S1fea/rwamAsMyve8HWVFvPRk5n/JMQReBGIvFj+PU4vWnQ1aB1qf+XKWo3Vnoc7qNp1DQrsDZbl3hr5XuLCmWyhoG7mIDANGA2/n2DdRRGaIyIyaGnviMO2gDWR+8dtwe7r1Cas6S+OzkGsiLHUg9nzbzhE5BAKb0Xw6hChEDkUCWxYipelGClbIRaQCeAy4SFVXbbhfVSerarWqVldVVRXqssZFqg4afxtt/Deamtdp15Hw/pmmi+w97sz/onVAPMeOJLRxql+RINJnClScD4GtIbgj0uNXSM/fFTSq6R4K0v1QRIJkiviDqvqvQpzTdG2aXozWnrauPVfTaORApOdNhR+SHtwRIodD7LmmJ3ABIlB2sjtd9UK7gtyT/duABNu1cLL4ypCKc6DinAIHNMWgyTno6t9D8gPwD0AqzkMiY13JUoheKwLcA8xV1Zvzj2S8QFf8FNILgPV6XsSmocEHkfIJBb2WiECP6yFyJNr4JIgfiY5H3BpBGRwDoV0g/jbQ2LQxCqF9MpOOmZKnyTno8pNY+/8/tRJdeSmaXo6v/NSi5xHV/KbYFJG9gFeBD1j3RupyVX22pc9UV1frjBkz8rqucY86tejSvcmsHrQB/3B8VW1sJ/Yw1RQ0Pok2Pgr4kLLjIPJ9myCrm3Bqz4HEf8maolgqkf5vkWmkKDwRmamq1Rtuz/uJXFVfI+dKBKZkaYzMIhM5CrnmWl2o9IgEoOwYpMzW4+yWUh+Qc555TWWaG/2bFjVOaYzsNMXlGwi+XNPaBiFyaNHjGFN0/pZ6WDuZmS+LzAq5aTcRQXrdmFk0ge9+hYw2vfA5181oxhSFVJxHzmlByo5HJFr0PJ6fNMu4Q0LV0O95tOFhSM+H4K5I2VGu/CU2ptgkvB/a4ypY/bvMgDAEyk5AKn/hSh4r5KbDxD8QqbzQ7RgmT+qsgdjTaOoLJDgisyBGzn77Zn2+svFodFxm7ICvEpGQa1mskBvjIeqszvSnd+oyfdaDo/KalkJTX6PLj296Sd2IShms/gP0fQzx28C9jcksvO3+MohWyI3xCE28h9b9sGl6gAQQgvBekMe6oLrqStCVrO05rA2gCXT1DUivm7KPT36G1k+G1McQ2A6pONumFOgCrJAb0wnUqc80VyQ/yzRXRA/P6/2BahpdcV5Te+x3mqa8jT0J0fEdOGcqs75o1oRkKYi/lH184n9o7RlkpidwIPUZGnse+jxgA6FcZr1WTElSTaCJ99HUFwU8Zyozoi/1Ja0NpNPUArTmQHTVb6HxAXTVNWjNIWh6cccvnprTQh/9RrTh0Q6eVGh5CEj2M56uuorMSMbvCr+Tuf7q6zp4fVMo9kRuSo7T+BSs+hUgoCnUPwTpfTcSGNzhc2p8OrriEiCZmXXRPwh6/wkJDMs+dtWvQFewruA1gBNHV12P9L61owk6uK9lIn40fADEX6b57JJBiHy/+RVUITU394mSH3To+qZw7InclBRNzoGVk5rmCV9DZgm1L9C6M1t9im71nKmv0bqfZIqz1rP+smxOYjZO7UScpXvjLD8VJ/4mJN4gu7kiDfFpHb+xwPY0n/L2O1EkemyHTys9rgL/YJByIARSBoGtkMqfNz9OpOmYXCep7PD1TWHYE7kpKdrwINkr7ziZYdPJ9yG0U/vP2fgI2fOhKzirofakpn0KzhKom93yifKYh0XED71va1rQwgHimal9Q7tCtONLGoq/L/R7DhKvQWo+BLaC0G65e8JET4GGvwLrN/FEoMCTpJn2s0JuSkt6CTlXE8IHzvIOnnMRORe2+O6lXzMxMiP+kjSbGZIgRI7o2PWbSGgMVE2D2LPrdT+szntVLBF/Zl73XA/86x9XeSGqy6DxKZAQaBKi30fKf5zX9U3+rJCb0hLer6knxgYvBjUJwfY/jQNIeG80/lKO1Yhy/cAASIBvKGhN5roSAP9QpPLSDl2/WRZfbyg7Je/zdOjaEkB6/hatvARS30BgCOLr40oW05wV8jb65pOFLPh0EcO2H8LA4QPcjmNaINFj0Ia/Qfpb1q3iE4XyMzPNCB0RGQv1f4HUV83PiQ+ozz5eItDvWST5FqTmtd5c0QpNL8706fYNRpwvAQH/5q6vSyu+PhCyAt6VWCHfiMb6GFcdcyMfvvYxgVCAZDzJmLGjmTTlIoKhzplz2HSc+Mqg72OZtvLY8+DriZSdhkQO7Pg5JQR9/pn5ARF7BqQMKTsZdRph9fWsW1yCzERiZRPw+QKZwTrhvdp9PU19ja64AFJfkOmRkkYJAL7MrJO9b0OC23f4fkzpyXthiY7w0sISt5xzF1MfeIVkfN3c2+FoiPEXHsEPrz/ZxWTGbaqK1t8N9XeBKqBNEydd2vGRlppEa/YHZxktNt1IBVL1CuKr6HB2400tLSxh3Q9boapZRRwg3pjg2T9PdSmV6SpEBF/Fj5H+7yD9nkL6v4Wvx6T8VgmKv9rUxbGl9ncyvVZipb8Kk2k7a1pphZN2SCVy9VaAWH2uVdRNdyQSgsDQwpzMWZoZcNSq2LpFr43Bnshb5Q/42bp6i6ztIsKo/Ua6kKjw5rz1KT/b91eM6zWBH438KdMffsPtSN1bsA1zlkgEQjt3fhbjGVbIN+LCP51NtCJCIJT55SUYDlLWM8q5t5zucrL8ffzOZ/zioKv54NW5NKxqZP6cBdx41p08+acX3I7WbUlwROYFaYsTbEUyxT44pqi5TP40vTS/+XZaUZCXnSJyL3AksFRVN/qo6qWXnQBLv67h8dueY97s+Ww7ZkuOOu8w+g4s/rp8hXbJgVcxa9qHWdsrepXz6NJ78AdsRXg3qKbQhinQ8BBoHPxDM00u4oPocUjZia4uYmDaR1NfoisuauqFBPiHIL1uQYLbtvtcLb3sLFQh3wdYAzxQioW8VB1TdSarl6/J2h6KBPn7l3fSe0AvF1IZUzpU4+jSfUHraDa5mfRAqqYhvvbNU9OpvVZU9RWgthDnMsUzYLPcK8CIz0d5rxYmSDLGtF1sKplBZBs+MCczYxIKpGht5CIyUURmiMiMmhp7494VnP6bEwiXNf8VPVwWZtz5hxEK22AnY/LmLM40j21IG9H0ooJdpmiFXFUnq2q1qlZXVdlagF3BbkfuzAV3nk2v/j0JhAJEKiKMv2AsZ113ktvRjCkNwR0zE4xtSMqQtvRQaiPrR97NHTJhPw46dR/qVzZQVhm1F5zGFFJwTGYu+eRs1s3TEwb/sMyMkwVihdzg8/mo7G3DvY0pNBGBPveh9fdA42OZUbnRcUj52fmNAN5AQQq5iEwB9gP6icgC4Neqek8hzm2MMV4mEkIqzoWKczvtGgUp5KpqjarGGOMSG9lpjDEeZ4XcGGM8zgq5McZ4nBVyY4zxOCvk3VA6lSad2tic18YYr7B+5N3IsoXLueWcu5n54vsA7HzIKC666xyqBndwUWJjNqCprzMzN6a/gtCuSPQ4W5KuCGzNzm4iEU9y+lbnU7toBU46s4yYz++jzya9+Ovnt9vcKiZvmngHrT0bSAFJIJpZ/Lrv44jfHhYKwdbs7Obe+Pc71K9sWFvEIbOUXf3KBl5//B0Xk5lSoKroil8AjWSKOJk/O8vQNbe7mKx7sELeTSz8bHHOdUYb62Ms/CwzC5tqEm18Eqfu/+GsvAxNzCp2TONVzhJwlufYkYL4f4oep7uxNvJuYtjIIUTKwjSuiTXbHq2IMGzkkEwRrz0dknOABkDQxmfQyp/iKz/TlczGQyQCOC3sa2nZOlMo9kTeTex25M703bQ3geC6iXoCQT99NunN7t+vhtjz6xVxyEyEH4PVN6NOHV9+MJ/LD7+eo/uczpnbXsgL903DjfcrpmsSXy8IVZP9bBiFslPdiNStWCHvJvwBP398/ToOOnUfopURopURDjx1H2594zr8AT8ae551RXw9EqDmy+e5YM8rmPHC/6hf0cCCT7/l9p/cw9+vfbTo92G6Lul5EwQ2BykDKQfCEDkUKTvZ7Wglz3qtGACclZMy02xu+OuxlPPQXQfy12u+xnGa/10Jl4V5ZMlfiJZHihfUdGmqmpl72/kWAtsjgaFuRyop1mvFtEqiPwByrcweZuqUhqwiDpnui0u+smX7zDoigoRGIZGxVsSLyAq5AUBCo6Dy50AYpCLzq7H0Qfrcyyabb5rzM6lEij4DexU3qDEmi/VaMWv5yieg0aMg8W6mkId2QSTAKZOE2dPnkIgl1x4biobY9/jd6dGn0sXExhiwQm42IL5eEDm42bbl39bhOM3bzoOhAOf8fkIxoxljWmBNK6ZVyUSSmyfeRSrRfJKtZDzJc3/xxkCPhtWNrFq+2u0YxnQaK+SmVfPen59zeyKWZPqjbxY5TfusqFnJpCOv59iqszhh0ETO2u4i5r79Wc5jVdX6xRvPskJuWhWtjJJO5R6xV9GrvMhp2k5V+cVBV/Pe1NmkEilSiRTffLyQXx58NTUL1g0lV2cFzoqL0SUj0SXb4dT+CE0tcDG5Me1XkEIuIoeJyCci8rmIXFqIc5quYei2gxi4eX/EJ822R8rDHH3+WJdSbdzctz5l8ZdLSSWbNwmlEimeuftFAFQdtPaUzKhWkkAaEq+htcejTn3xQxvTQXkXchHxA3cAY4HtgJNEZLt8z2u6jquf/CWbDOtPtDJCWY8owXCQo88fyx7jxrgdrUWLv6oByd6eTKT45pNvM98k3oT0t6ybrQ/AAacBbXy6GDGNKYhC9FrZBfhcVecBiMhDwDhgTgHObbqAgZsP4K+f3cZHb3zCiqUr2W73remzSW+3Y7Vqy9Gb52wSCpeFGLnXtplvUvNAUzk+3QipTzo3oDEFVIhCPgj4Zr3vFwC7bniQiEwEJgIMHWojvrxGRBi557Zux2izodsOYpexo3n3uf8Rb0wAmflmKnqVc8gZ+2cOCmwJ4s/MD9ZMGRIcUdS8xuSjEG3kOX6Bzf6noaqTVbVaVaurqqoKcFnvmDd7Ps/+5T+8+/z/SKdtrcximTTlIk698jj6b1ZFz6oeHHz6vtw543eU9yjLHBDaFfxDgfVXR/KBrxwiR7gR2ZgOKcQT+QJgyHrfDwa+LcB5PS+dSnP1D25i5ovvIwjiFyp7V3Dz9KsZsFn3+mHmhkAwwImXjufES8fn3C/igz5/R1f/FhqfAdIQ3g/pcSXiKytuWGPyUIgn8neBrURkcxEJAScCTxbgvJ7379ueZeaL7xNvSBBriNO4OsayBcu59sRb3I5mmoivB76ev8W3yWx8m3yEr/cdiH8Tt2MZ0y55F3JVTQHnAy8Ac4GHVfWjfM9bCp6Z/BLxhkSzbY6jfDHrS+qWrHAplTGm1BRkrhVVfRZ4thDnKiWJeDLndvH5SLawzxhj2stGdnai/X6wB8Fw9s/KvgN7UzWkX4ufq11cxzOTp/L03VNZ9m1tZ0Y0xpQAK+Sd6KTLxjNgWH8i5WEAQpEg0YoIl/7tJ4jk6uwDz9/3MqcNP48//ex+7rr4fk7f8nyenjx1o9dSVeKNcZsvxJhuyJZ662SJeJLpD7/B7OlzGDh8AIedtX+Lg2lqFiznjK1/0mzeb8jM/X3PR7ewybD+WZ9xHId//u7f/PPGJ2hcHaPPwF5MvHEC+5+wZ6fcjzHGPS0t9WbzkXeyUDjIwafty8Gn7bvRY1997K2c2520wyuPvMkPLhmXte/Bax/jn//3BPGGOADLFtRy0w/vpKwiwq5H7JxfeGOMJ1jTSheSTqbRHGtjqqM5h5unU2ke+f2Ta4v4d+INCe7/9T87LacpbaqNOKtvxFm6B86SXXBWXok69q6mK7NC3oXsPm4M4s/+XxII+tnj6OwJqtasqM+a3e87i+YtafE6q5av5vn7pvHM5KnNpnQ1RlXR2rOg/gFwloGugMbH0OXHoRrf+AmMK6yQdyGDtxrIyZcfQzgawuf34fP7CEdDHPfz77PZiMFZx1f0LidcFsp5rmHbD8m5/dV/vc3JQ3/MHRfey59+dj9nbP0THr35qbX7axfXsXLZqsLckPGe5P8gNRdYv2inwKmF2HNupTIbYW3krahf1cB9V0xh2kOvg8J+J+zBmdee1KkLKpwy6Vj2OKqa6Q+/iaqy93G7seVOm+c81u/3c8bVJ/DnXz7YrHklHA1x1nUnZx2/qnY1N5x2K4nG5oOU7r/yIfoN7svfrnqYRfOWoApbfW84l//jwpwvWE0JS84BzfFbnjagifeR6NHFz2Q2ynqttCCdTnPu937BN598SyqRmeo0GAowcItNmPz+7/EH/C4nXGfq36bzt6sfYfnCWjbbfggT/+80dtp/ZLNj5s9dwK/G/Y5vP1+c9XnxCf6Af+19Avh8Qp9Ne/P3eXd2qXs1nUvj09EVF4FuuLBGBCovwVd+miu5TIb1WmmnGS+8z+KvljYrbslEippvlvH2M+91qUUVNtYrZuWyVVy05xWsWZF71Rt1FMdp/jLVcZT6lY2889z/2P37WX9vTKkK7QW+3pCOAd89mQtICIlm95oyXYO1kbfgi1lfEa/PfrnTuCbGF+9/VfxAeXj+3pdbnC4AwB/w4eTqFZNMUfONvQztTkT8SJ8pENqdzHNeAAIjkb4PIb4ebsczLbAn8hYMHD6AcFmYxjWxZtujFREGDh/gUqqO+fKDr7Paxb8TCAXYcd/tmPvmp1n36vP72Lp6eDEimi5E/AOQPvei2giaRnwVbkcyG2FP5C3Yc/wuRCuj+NbrDujzCeGyMHsfm7UAUpe2zZgtCJeFs7b7Aj4uumsi1z51Kf0G9202L0woGmL7PbZh2122KmZU04WIRK2Ie4QV8haEwkFufeM6dtxnBP6AH3/Az8i9R3DrG9cRjmYXxa7skNP3IxzN7qYoIuyw9wiCoSC3vXkd484fS9XgvgwcPoBTrziWa5661IW0xpj2sl4rbRBr6toXyfFU6xVXjruBt56e2WwRPvEJo/bbnhtf+rV7wYwxbWa9VvLg5QL+nVkvf5i1kqo6yuzpc0in0tbF0BgPs6aVbsKXY+g/ZJpXxJd7Sl1jjDdYIe8mDjx576xFLvxBP7sfVY3PZ38NjPEy+xfcTfzwhlMYNnIokYpIZoGLykw3ygv/dLbb0YwxebI28i4m1hDn7adnsmr5akbtP5Kh2w4qyHnLe5Rxxzs3MHv6HL788GsGb70p3ztoB3saN6YE5FXIReR44DfACGAXVe20riipZIrXH3+H96d/xIBhVRwyYT96D+jVWZfLaeWyVSyZX8PA4QOo7F34/rWfzPiCXx5yNU7awUk5KHDQqXtz0V3ntLg0XHuIZHqpjNpv+/zDGmO6jHyfyD8EjgHuLkCWFjWuaeTCPa9g8ZdLaVwTIxQJ8uA1j3HDC1ew3e7btP7Z+lhmIE8efb9TyRS3nHM3/33odQKhAMlEiiPOPohzbzmjYE+0juPwq3E3UL+iodn2l//xGtWH7MTex+5WkOsYY0pPXlVIVeeq6ieFCtOSh3//JAs/W7R2CHkilqRxTYzrT/5ji4sNf/3xQi7Y/XLG9z6DcT1P5/Kx17F8UV2Hrn/vpClM/+cbJGJJGlY1kowlee6el3n0pqc2/uE2+nTGF1lD5AFi9XGe+fNLBbuOMab0FK2BVEQmisgMEZlRU1PTrs9Om/J61oLEACtqVrL4y6VZ29esqOfCPSfx8TufkU6lSafSvPef2fx07ytJp3OvqNMSVeWpP71AfIO5SuINcR675el2nas1qUSqxeaTZCsTXhljzEYLuYi8JCIf5vhq15yWqjpZVatVtbqqqqpdIYOh3C1A6iiBHPv+8+ArJOMp1n9YT6ccVtSsZOaLs9t1bSftEG/IPeHU6ro17TpXa7bZZcuchTxcFm7Tws3GmO5ro4VcVQ9S1ZE5vp4oRkCAwycelLWkmfiEIdsMompw36zjv/l4YdaCxJBZ3Li1tSxz8Qf8DBmRu+fINmO2bNe5WhMMBbns7xcQLgut/cEVqYgwYtetOOi0fQp2HWNM6fFE98Ojzj2UWS9/yMyps0EVf8BPpCLClY/8LOfx24zZikjFdGI5pmXdYtRm7b7+Bbf/iElH/pZELIE6is/vIxQJcu4tZ3Tkdlq06xE7c+/cPzL1genULVnJmENHMWbsaOsiaIxpVV6TZonIeOA2oApYAcxS1UM39rmOTpr1+awvmfvmp/Qd1Iddxo4mEMz9cygRS3DWdhexbGEt6aZV5kORIFt+bzh/ePWaDnXl+3zWl0z57eN89eE3bLXzcE66bHzOBZFN1xJvjKNaGvPlGNPSpFklO/vhipqV3HP5P3jtX28TCPg5+PT9mPCbH9g/6G5i6TfL+P1ZdzJ7+hxAGbHb1vz83v/HoC0Huh3NmA7rdoW8u0un07z55AzeenomPfpWcNhZBxZslGhXl0wkmbDlT6hdVIeTzixhJz6hR99K/jbvDqLlEZcTGtMxNo1tN5JKprjssOv4+N3Pia2J4Q/4efKOF7ho8jkcdErpvzh96+n3qF9Zv7aIQ6aHU7wxwfSH3+SwM/d3MZ0xhWdv0UrQtIde5+N3Plv7sjedShNvTPCHcybTWJ896KjULPpiMckc4w5ia2Is/HyRC4mM6VxWyEvQtIdeJ1af3f3SH/Dx4Wsfu5CouLbYaRjBcDBre7QiwpY7be5CImM6lxXyEhQtb+GFrpJz7c5SM/rAHRi01cBm868HQgH6DOzNnkePcTGZMZ3DCnkJOmLiwURyFPNQJMj2e7Y+yVgp8Pl83PTfqzjynEPo0a+Syj4VHHrG/tz65nUtdlk1xsus10qJunfSP3jslqfxB3yI+PAFfNzwwq0zgLEAAAmsSURBVJVsU72F29GMMR1k3Q+7oaXfLOP9aR9R3rOM6sN2IpSj3dgY4x3W/bAb6j+kHwdPsAm3jCl11kZujDEeZ4XcGGM8zgq5McZ4nBVyY4zxOCvkxhjjcVbIjTHG46yQG2OMx1khN8YYj7NCbowxHmeF3BhjPM4KuTHGeFxehVxEbhSRj0Vktog8LiK9ChXMGGNM2+T7RD4VGKmqOwKfApflH8kYY0x75DX7oaq+uN63bwHH5RfHmK6lbulKXrhvGgs+WciI3bbmgFP2JloecTuWMc0UbD5yEXkK+Keq/r2F/ROBiQBDhw7def78+QW5rmndqtrVBEMBohVRt6N4zuezvuTi/X5NKpEiEUsSKQ/To28lt79zA73793Q7numGWpqPfKNNKyLykoh8mONr3HrHTAJSwIMtnUdVJ6tqtapWV1VVdfQ+TBt99t48zt7xYk7YdCLH9D2Tyw67ltrFdW7H8pQbz7yDhlWNJGJJAGL1cZYvquO+K6a4nMyY5vJ+IheR04EfAweqakNbPmMrBHWuuiUrOGPrC2hY3bh2mz/gZ+AWA7jno1vw+ayz0sasWVHP8QN+SCqZztrXo18ljy2914VUprvr8BP5Rk56GPBL4Ki2FnHT+Z6792VSyVSzbelUmuULa5k9fY5LqbzFH/S3uM+WzDNdTb6PZrcDlcBUEZklIncVIJPJ0zeffLu2OWB9qsrir2pcSOQ90fIIow/YAX+geUEPRUOM/dGBLqUyJre8CrmqbqmqQ1R1p6avHxcqmOm4kXtsQ6Q8nLVdFbb63uYuJPKmS+4/j4HD+xOtjBAuCxMuC7PDXtty4qXj3Y5mTDO2+HIJOuCUvfnH9f8imUiRbmrjDUdD7Ljvdmwxapi74Tyk94Be3DPnD7z/349Y8lUNW+w0jK2+N9ztWMZksUJegqLlEW5/5wbuu2IKbzzxLqFIkMN/dCAn/PJot6N5js/nY/QBO7gdw5hWFawfeXtYrxVjjGm/Tum1Yowxxn1WyI0xxuOskBtjjMdZITfGGI+zQm6MMR5n3Q+Np62uW8NbT88knUyzy+Gj6bNJb7cjGVN0VsiNZ732+NvccOqt+Pw+VJXbzneYeONpjDtvrNvRjCkqK+QtmPPmJzxy01Ms/XoZOx+8I+MvPMLmoO5CVi1fzQ2n3kq8MdFs++Rf/J3RB+7I0G0HuZTMmOKzNvIc/vOPV/nFwVfz+uNv8+mML3j05qeYuOPFNp93F/LGE+8iPsnank6mmTblNRcSGeMeK+QbSCVT3H7+PcQbEnw36DUZT7Gmbg1Trn/c3XBmrWQihTrZo5KdtEMilsjxCWNKlxXyDSz8bBHpVPZiAqlkmndf+J8LiUwuux4+mlzTS4SiIfYcv6sLiYxxjxXyDVT2qci5KgxAzyprI+8q+g+tYsJVJxCOhvD5fYgIkbIwB0/Yl+1229rteMYUlb3s3ECfTXozcq9t+ODVuaQS6wp6pDzM8Rd/38VkZkMnXDKO6kNG8fKDr5JMpNjn+N3Zfo9t3I5lTNFZIc/hiod+xq/H/x+fzZxHIBQgGU9x4qXj2ct+Ze9ythg1zOZYN92eFfIcevSt5JZXrmHh54uoXbSC4TsOpbxnuduxjDEmJyvkrRi05UAGbTnQ7RjGGNOqvF52isg1IjK7aeHlF0Vk00IFM8YY0zb59lq5UVV3VNWdgKeBXxUgkzHGmHbIq5Cr6qr1vi0Hir9unDHGdHN5t5GLyHXABGAlsH/eiYwxxrTLRp/IReQlEfkwx9c4AFWdpKpDgAeB81s5z0QRmSEiM2pqagp3B8YY081JrmHOHTqRyGbAM6o6cmPHVldX64wZMwpyXWOM6S5EZKaqVm+4Pd9eK1ut9+1RwMf5nM8YY0z75dtGfoOIbAM4wHzgx/lHMsYY0x55FXJVPbZQQUz7JOJJ7r9yCs/++T/EGxLssM8Izrv1LDYbMdjtaMaYIrPZDz3q2hNu5ok7XqB+ZQOpZIpZL3/ABbtfzrJva92OZowpMivkHrTw80XMnDqbxHrLnKlCMp7kyTuedzGZMcYNVsg96Ou5CwkE/Vnbk/EUn874woVExhg3WSH3oMFbDySdY/GLQCjAFjtt7kIiY4ybrJB70JBtBrHDPiMIRYLNtgfDQcadf5hLqYwxbrFC7lG/+dclHHrmAYSjIcQnbLfHNtzyytX0H9LP7WjGmCIr2MjO9rCRnYWjqqgqPp/9TDam1LU0stMWlvA4EUFE3I5hjHGRPcYZY4zHWSE3xhiPs0JujDEeZ4XcGGM8zgq5McZ4nCvdD0Wkhsy0t+3VD1hW4DhdSanfH5T+Pdr9eVtXv7/NVLVqw42uFPKOEpEZufpQlopSvz8o/Xu0+/M2r96fNa0YY4zHWSE3xhiP81ohn+x2gE5W6vcHpX+Pdn/e5sn781QbuTHGmGxeeyI3xhizASvkxhjjcZ4r5CJyo4h8LCKzReRxEenldqZCEpHjReQjEXFExHPdoFoiIoeJyCci8rmIXOp2nkITkXtFZKmIfOh2lkITkSEiMk1E5jb93bzQ7UyFJCIREXlHRN5vur+r3M7UXp4r5MBUYKSq7gh8Clzmcp5C+xA4BnjF7SCFIiJ+4A5gLLAdcJKIbOduqoK7HyjV5ZlSwMWqOgLYDTivxP7/xYEDVHUUsBNwmIjs5nKmdvFcIVfVF1U11fTtW8BgN/MUmqrOVdVP3M5RYLsAn6vqPFVNAA8B41zOVFCq+gpQ63aOzqCqi1T1vaY/rwbmAoPcTVU4mrGm6dtg05eneoF4rpBv4CzgObdDmI0aBHyz3vcLKKFC0J2IyDBgNPC2u0kKS0T8IjILWApMVVVP3V+XXCFIRF4CNsmxa5KqPtF0zCQyv/I9WMxshdCW+ysxuZYw8tQTjwERqQAeAy5S1VVu5ykkVU0DOzW9c3tcREaqqmfed3TJQq6qB7W2X0ROB44EDlQPdoTf2P2VoAXAkPW+Hwx861IW0wEiEiRTxB9U1X+5naezqOoKEfkvmfcdninknmtaEZHDgF8CR6lqg9t5TJu8C2wlIpuLSAg4EXjS5UymjSSzKOw9wFxVvdntPIUmIlXf9X4TkShwEPCxu6nax3OFHLgdqASmisgsEbnL7UCFJCLjRWQBsDvwjIi84HamfDW9nD4feIHMi7KHVfUjd1MVlohMAd4EthGRBSLyQ7czFdCewGnAAU3/5maJyOFuhyqggcA0EZlN5qFjqqo+7XKmdrEh+sYY43FefCI3xhizHivkxhjjcVbIjTHG46yQG2OMx1khN8YYj7NCbowxHmeF3BhjPO7/A6HyI7CzfdLiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_train, y_train = create_toy_data()\n",
    "x1, x2 = np.meshgrid(np.linspace(-5, 5, 100), np.linspace(-5, 5, 100))\n",
    "plt.scatter(x_train[:, 0], x_train[:, 1], c=y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import itertools\n",
    "import functools\n",
    "\n",
    "class PolynomialFeature(object):\n",
    "    \"\"\"Polynomial features\n",
    "\n",
    "    transforms input array with polynomial features\n",
    "\n",
    "    Example 1\n",
    "    =========\n",
    "    x = [a, b]\n",
    "\n",
    "    y = PolynomialFeature(degree=5).transform(x)\n",
    "    y = [[1, a, a^2, a^3, a^4, a^5],\n",
    "         [1, b, b^2, b^3, b^4, b^5]]\n",
    "\n",
    "    Example 1\n",
    "    =========\n",
    "    x = [[a, b],\n",
    "         [c, d]]\n",
    "\n",
    "    y = PolynomialFeature(degree=2).transform(x)\n",
    "    y = [[1, a, b, a^2, a*b, b^2],\n",
    "         [1, c, d, c^2, c*d, d^2]]\n",
    "    \"\"\"\n",
    "    def __init__(self, degree=2):\n",
    "        \"\"\"Construct polynomial features\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        degree: int\n",
    "          degree of polynomial\n",
    "        \"\"\"\n",
    "        if not isinstance(degree, int):\n",
    "            raise ValueError(\"degree should be int type\")\n",
    "        self.degree = degree\n",
    "\n",
    "    def transform(self, x):\n",
    "        \"\"\"Transforms input array with polynomial features\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        x: ndarray with shape (N, D)\n",
    "           input array\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        output: ndarray with shape (N, K+1)\n",
    "           polynomial features\n",
    "        \"\"\"\n",
    "        if x.ndim == 1:\n",
    "            x = x[:, None]\n",
    "        x_t = x.transpose()\n",
    "        features = [np.ones(len(x))]\n",
    "        for degree in range(1, self.degree+1):\n",
    "            for items in itertools.combinations_with_replacement(x_t, degree):\n",
    "                features.append(functools.reduce(lambda x, y: x * y, items))\n",
    "        return np.array(features).transpose()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabelTransformer(object):\n",
    "    \"\"\"label encoder decoder\n",
    "    \n",
    "    Attr\n",
    "    ====\n",
    "    n_classes: int\n",
    "        number of classes\n",
    "    \"\"\"\n",
    "    def __init__(self, n_classes=None):\n",
    "        self.n_classes = n_classes\n",
    "\n",
    "    @property\n",
    "    def n_classes(self):\n",
    "        return self.__n_classes\n",
    "\n",
    "    @n_classes.setter\n",
    "    def n_classes(self, K):\n",
    "        self.__n_classes = K\n",
    "        self.__encoder = None if K is None else np.eye(K)\n",
    "\n",
    "    @property\n",
    "    def encoder(self):\n",
    "        return self.__encoder\n",
    "\n",
    "    def encode(self, class_indices):\n",
    "        \"\"\"encode class index into one-of-k code\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        class_indices: ndarray with shape (N,)\n",
    "            non-negative class index\n",
    "            elements must be integer in [0, n_classes)\n",
    "        \n",
    "        Returns\n",
    "        =======\n",
    "        output: ndarrray with shape (N, K)\n",
    "            one-of-k encoding of input\n",
    "        \"\"\"\n",
    "        if self.n_classes is None:\n",
    "            self.n_classes = np.max(class_indices) + 1\n",
    "\n",
    "        return self.encoder[class_indices]\n",
    "\n",
    "    def decode(self, onehot):\n",
    "        \"\"\" decode one-of-k code into class index\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        onehot: ndarray with shape (N, K)\n",
    "            one-of-k code\n",
    "        \n",
    "        Returns\n",
    "        =======\n",
    "        output: ndarray with shape (N,)\n",
    "            class index\n",
    "        \"\"\"\n",
    "        return np.argmax(onehot, axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeastSquaresClassifier(object):\n",
    "    \"\"\"Least Squares Classifier Model\n",
    "\n",
    "    Params\n",
    "    ======\n",
    "    X: (N, D)\n",
    "    W: (D, K)\n",
    "    y = argmax_k X @ W\n",
    "    \"\"\"\n",
    "    def fit(self, X, t):\n",
    "        \"\"\"Least squares fitting for classification\n",
    "\n",
    "        Params\n",
    "        ======\n",
    "        X: ndarray with shape (N, D)\n",
    "           training independent variable\n",
    "        t: ndarray with shape (N,) or (N, K)\n",
    "           training dependent variable\n",
    "           in class index (N,) or one-hot coding (N,K)\n",
    "        \"\"\"\n",
    "        if t.ndim == 1:\n",
    "            t = LabelTransformer().encode(t)\n",
    "        self.W = np.linalg.pinv(X) @ t\n",
    "\n",
    "    def classify(self, X):\n",
    "        \"\"\"classify input data\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        X: ndarray with shape (N, D)\n",
    "           independent variable to be classified\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        output: ndarray with shape (N)\n",
    "           class index for each input\n",
    "        \"\"\"\n",
    "        return np.argmax(X @ self.W, axis=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD4CAYAAADxeG0DAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd3hc1Zn48e+ZrmpJlpssWe42btjGNhBMLzE9oSwBsiHAxkAgSwrL0rL5bZZfEtLYUJaEEEg2sGQhgUDomBac2IC7LeOGm+QqWZasNvWe/eOqz4wla2Z05868n+fRg3Vn5s7rYfzOmXPPeV+ltUYIIYR9OawOQAghRGIkkQshhM1JIhdCCJuTRC6EEDYniVwIIWzOZcWTlpYW6zGVZVY8tRBC2NbqVRvrtNbDeh+3JJGPqSxj6bJnrXhqIYSwrTzv8btiHZepFSGEsDlJ5EIIYXOSyIUQwuYsSeSNwWYO+vdY8dRCCJFxLEnkLS0ulmzfwqpDG6x4eiGEyCiWJPJCp5eD66awpMrg9erlMjoXQogEWLL80OtxceXsaSyt2sWyAweoHrOOirJqThhWwXDfaCtCEkII27IkkXdYOL2SipoCPt6Rz7KD1RyesIWpIw4zd+gMK8MSQghbsTSRA1SWl1BZXsLSqjyqqg7Q3PQZB1qbZXQuhBD9ZHki7yCjcyGEGJi0SeQgo3MhhBiItErkHeKNzsvzinvcT5K7EEKkaSKH6NF5TZ2foqK2ztvzXdUy9SKEEKRxIu/QMTqvrm2C2q7jVYYhUy9CCIENEjl0jc67kwujQghhskUij0UujAohhMm2ibyDLFsUQmS7pCVypZQTWAHs0VpflKzz9kfH6HxXzUg+XrefmrI6DoxbLqNzIURWSOaI/HbgU6Awiec8Jl3TLWYNl8PTZXQuhMh8Sal+qJQqBy4EnkjG+RK1cHoli4onSoVFIURWSNaI/D+BO4GCeHdQSi0GFgOUjUr9dIeMzoUQ2SLhEblS6iLgoNZ65dHup7V+XGs9T2s9r6S45Gh3TapYo3NpaCGEyCTJGJGfAlyilLoA8AGFSqmntdZfTsK5k6L36Lx8TNdSxe7kwqgQwo4STuRa67uBuwGUUmcAd6RTEu+u91LF6krZ8i+EsD/bryM/Vt03ErG16/i+UC41ZXVsKn2PqSOGSUIXQthGUhO51vp94P1knjNVFk6vjDq2tGoXVQd8sktUCGErWTciPxrZJWotHTkEgffAOATuWeCZj1LyFhWiL/KvpBep4WINHVwPTT8AbQAhCHwAztHoIfejlNfq8IRIa0nZEJSJOpYttu44nmVrFEu2b5FliymitYbmB0EHgFD7QT+Eq6HtNUtjE8IOZER+FDI6HySRajNxRwlC8K+Q+8VBD0kIO5FE3g8yd55qrvYplVjcgxqJEHYkibyfZHSeQs5R4CyFyD5Ad7vBC75FVkUlhG3IHPkx6jF3vrxE5s6TQCkFBf8KqgBUDuAxf7wLwHum1eEJkfZkRD4AXfXP6/l4Xb7UP08C5apAl/wagqtBN4BrKso1xuqwhLAFSeQJkAqLyaWU2xyFCyGOiSTyJOi8GNprdN6djNSFEKkiiTxJeo/Oq6e09bi9olimXoQQqSGJPMk6RufVW5t6HF9mGDL1IoRICUnkKdAxOu8u1tSLjM6FEMkgiXyQxLswWp5XLAldCJEQSeSDrPuF0U151UydUCvTLUKIhEgit4DsEhVCJJMkcgtJDRchRDJIIreYjM6FEImSRJ4mZHQu+kOHP4PWF8DYC67J4LsM5RphdVjCYpLI04iMzsXR6OBqOPIAZvMNbTbeCCxFD/kRylXR18NFBpNEnobijc7L84p73E+Se/Ywuyj9Egh2O2qYDTlafw+F91gVmkgDksjTVO/ReU2dn6Kirm3/+a5qmXrJJroFjMOxboDQxkEPR6QXSeRprnPLf20T1HYdrzIMmXrJJsoLqNi3OQoGNRSRfiSR20DcLf9yYTRrKOVG+04H/wd0NqgGzC5Kl1oVlkgTkshtSi6M9k0bbRCpAUcJyjnU6nD6RRtHoPV5CH4MDh94LwTfuWYXpbx/AqMFgitAuUCHzVZ4vs9bHbawmCRym+uceqkZwbLd0twC2i8Mtj0PrX/qTHjaPQsKvo1y5FgdXlzaaIOG74DRAETAAMK/gchWyL8VpTxQ+C/oyGEw6sA5CuXItzpskQakZ2cGqCwv6ewlenDdFJZUGbxevZyD/j1Wh2aNwFJoexEIgW4z/xtaB82PWB3Z0QWWgNEIRLodDIH/XXSk6wKJchaj3JMkiYtOksgzSGV5CVfOnkZlbWV2N4b2vwg60OtgCIIrzFFvugquBcIxbtDmh5MQccjUSgaK13oua+bOjcY4NyjQzUC6Tq/o+DeFPxu8MITtyIg8Q8Uanfd3ukUbfnRwLTq02Zxvthv3TGIu1XPkgCONL3q6Z8W/TaZRxFHIiDzD9a5/3tdSRe1/D5ofB9X+Ga9y0IXfRbkqBzHqBOVcDcGV5q5HIphJ3QO5X0OpNB67+M6E1meInl7xgOcUKyISNiGJPAscbalid8NcIWj+FRDs+pav2+DI/0MXP4FSzkGPfSCUawS66EHzgmd4IziGQ85lKPcUq0M7KuUoROd/DZqfwPwA0oAHfAvBnb2rkETfJJFnkd41XKore275L/JUMTffyYje7wodhNAG8Bw/uAEnQDlLIf9rVodxzJTvXLRrOgT+ar7u3gXgmmKuIwe0Ni/aYhwC1yRwTe68TWQvSeRZpvvonK1dx/eFcvnUW01DZTlThh5ibl59zwcaTYMbaJrThh+UOyXfUpSrDFxfin7O8H44cq85ZaTDgAPck9GF96GUO+lxCPtIOJErpSqA/wZGYm5heFxr/YtEzytSa+H06DnvpWur2bCxiaZKDweG5TG36CAjXAEzabinWxBl+tGhjdD8GET2AU607wzIu9HcrJPoucMHgCPgrIx9vuaftW8W6nYBOrQZ2v4CuZcl/PzCvpIxIg8D39Far1JKFQArlVJva62lJJvNnDLrbEZvXs4n20pYVltMw1gfU4Y2MnfYmShncd8nyHA6XA2N36erlKwB/vfN5Fp498DPG2mEpgfMJYbKBWh07ldROed1u08DhHcTvUQxaG4kkkSe1RJO5FrrfcC+9j83KaU+BUYDkshtRikXY6fcy9jA+yytWkfVptE0jTM46KjkBMee7FmHHo//z/QsWIX5e3ANOlKLcg4b2Hmbfti+TjwCuv38LU+hXaNRnd+EIvEeDfoot4mskNQ5cqXUWGAO8FEyzysGj1Iu8J3DqSecw5iaej7etZ9ldZlTYVFrDZEd5ijaNRHlKOz/g8M1xNy0o9xgHIQBJHId3gfhnUQn6gC0vdw5paWcQ9HO4RDpvQ/ADd7Tjvl5RWZJWiJXSuUDfwK+qbU+EuP2xcBigLJRWT6ys4lMq7CoI4fgyL9DpA6UMotp5XwBlXd1/07gmgThHUSt89YhcAzw9dCNoJyxN3Uah3r+XvBtaLyvfQQeBOUDp7m0Mttoow1anzZX92CA92TI/cqxfTBnkKQkcmVeMv8T8IzW+oVY99FaPw48DjBz+iwbbhfMXhnTGLrphxDZi9kirf1Y28to1wSUd0Hfj8+5FALvtSfSjhN4wXcaylk0sJiclXGmRtzgPqHHEeUahy7+JQQ+hMgBcE8Fz3zzW1QW0VrDkfvMnqUdH6r+DyBUhS56KCtX8CS8zU2Zi1h/A3yqtf554iGJdNSx5X96cCqbqkbbrsKiDu9vnxoxet0SgLZX+nUO5RwGQ34E7jnmaNhRArlXQd7NA45LOXIg71rA2+2o29ySn3NhjPsXoHIuQOVfj/KenHVJHDArWUb20fObUcSssRPIzlndZLwLTgH+EVivlFrTfuwerfVrSTi3SDMLp1eykEqWVu1i2QE71T9viT+FQf/XyCtXBQy5L2lRAaici9HOCmh7CXSDORLPuSRrpwn6FNnZdVG4O+1vn/paONgRWS4Zq1aWEreZoMhU8SosdpdW8+jOMcR+m7rBfdJgRxNFeWaDZ7bVYdiDYxQoT3ut+e584Eqj99wgysLvZSJZui6GmqPz6ik9/2FVFKdP+Vyl3Oi8m6D5UcwlhO11TJxFkHORxdFZR2sDQmvBqDVX8bjGWx1S3zwngMpvrznfMVWmzOTu+ZyVkVlGErlIWGe7ua09pyiWGUZaTb0o36lo12jwv2quXHGfYPbDTOP2b8dC6xAElkN4Daih4D0H5Roe//6RQ+aWf6Op84Krdk+Hwrs6LxhqHQRUWl1AVMqJHvJD80M5tNY86DrObIfn8FkbnEWUFfWmZ06fpV94tn8XmIR97aqp5+O6/QTL6pg5rjVtRud2obUBkYPgyO+zrZvWAWi8B8L7AD/mGM0Bhf+K8syJ/ZiG70L4U3peAPZA7hVm2dzmRyG8CVBmjff8W9OuibVunytPpw+aVMrzHr9Saz2v93EZkYuU6T31kk6j81TRxhEwWsE5vF+1z7XWZiNl5UM5CrqO+z+ElifMCogYaM8cyL89/reHttcgvIeu8gHtKzqa/hNd8lRULNpohvBmolfxBKHtbXMlj26i8+pwaB003oMufjStVspkSwLvS/r8HxEZKxtaz2mjGZoeNMv94gDlQefdjPKdHP8xwdXmqNdoBgy0ewbk3QSR3e2NorutzAiuhqafwpDvxj5ZcCldSby7MER2gWtcryfvaLgRSzNoTc8lPobZJi+4Erwnxv07CWtIIheDIurC6Jh1VJRVZ05CP/IjCG+hcySsA9D8C7RrGMo1MeruOlwNR34MdGsSHVoLDV/vuEevR4QgtAEdORRnesMb4xjtCTu6kqJyDkE7R0KkutctLnCOgvD2GOcKgnEg9vMIS6Vx3yuRiRZOr2RR8URadxzPsjWKJdu3sOrQBqvDSohZL2Ur0S3aQma9lFj8rxFdgEt3+4lBucCoj32bbxHRyVyBsxScZbEfU3A7qBw6E73ygXOoeS4VYwpHuc2dqCLtyIhcDLpMq+GCrjeTXNQmFW1upY+lo1TAMT1PBJzlsW/zngrhDeZWdeXEXI7ng4K743YQUq7x6OLHwP8OGPvBNRW87Ztp2l6ASJCuYl5uc6R+tAbRwjKSyEVSHKlv5oP//Rvb1+1m1PjhnHn1QoaVH32FQ8bUcHFWxt5piDt+4nPPhNAmokfl8Xgh97Koi506vB3877VvWdfgOx3UCHCPA/fxfXYwUo5CyP1i1HE95IH2olR/BxzmB0XutdJWLk1JIhcJq605xANfeZigP0Q4FGbbmh0se3kF33j0nxg/6+hfxTNhdK4c+eicS9prtnTMeTvNEbEvzmYj3+fN6RWjifi1xhXmbsVR4PsiyndKj1t16/PQ+kd6fBiEFObo+Z8TakOnHAWQf4v5I9KeJHKRsBcfeo22Zj8dexIi4QiRcIT/+cEL3PeHb/XrHPFG5+V5PTsTpW1yz73GLAPgf8lMzp7ZkPMPKOeQmHdXjgL0kJ9B23NmM2XtN386k7rD3L1Y/FDMmis6vD86iZu3AEFoeQTtnS/L87KEJHKRsE0fbSPWxrIDO2sJtAbw5sZZUdFL79F5TZ2foqKubf/5ruqjTr1ordm5oZoDO2sZNWEEldPizCengFIKfKeaP/19jLMY8m8CbjJfP//r5q5T3QqeuZBzTfzCWaGVfZ3d7Ofpsdk0lRgQSeQiYb48L/5Wf9RxpRRO97F/ve/c8l/bBLVdx6sMI+7US1tzGw/d+gT7tx/sPFYxtYxbH7oRb07ijZFTTSkFOReYP/16gJujLzrToPr3AZqJtDaASNZ8I5FELhJ22hUn8cZT7xL0d33Nd7ldzD1nJi73wN5iHaPz7o52YfT5n/yFPVv2Ew51LQHcWVXDnx9+javu/MKAYkhrnpOAJ+PfrnIhxvr1TKd1CFqeMlfiEEY7yyH/ZpT7OKtDSylZRy4Sdt51Z3D8GTNweVzk5PvweN2MP76SL90VvRoiEb2bW6zYVs3r1cs50FbDirfX9kjiAOFgmI9eXZ3UGNKFchRC/u2AG+j+rccNqgAK783OFSZNP29P4u0VLiPV0Ph9cwNWBpMRuUiYw+ng+v/4Epfeuoi9n+2ndPRQRo4dYEf5foh1YVTNVbA8+r69k3smUb6T0Z6Z5rZ5o8HcMOQYAZ7js2ZKoTsdqYPgKqIvAIfMi9D5t1kR1qCQRC6SpmRkESUjB9i78hh1TL3sqhnJx+v2c2jkAQrP24pvawv+HeZ9HA7FcSdOGpR4rKIc+ebacWGWD4i5Mcto7++ZuSSRC1vrSOi5jRGeWxph8pwaXMObcawFj8/DlXdcYnWIYrA4RsfZmOXM+OsFkshFRrjw9JlMGDGUF5evpmn8fuYvCnPR5+ZTOrSk7weLjKCcRWjfaeD/kK5KkO2dg3yXWhlaykkiFxlj6tQy7p5axtKqXVT5D7D0wC6m0mq/Lf9i4PJuAcfI9l2zLeA+DvJuOGqnpEwgiVxknFj1z0fk5ktCzwJKOSD3cvMni0giFxmpd/3z8jFdG4m6S9st/0IcA0nkIqP1XqpYXdn/Lf9C2IUkcmF7+7YfYNuanRQU5zH9lKm4PT3f1t1ruLC12+NCuTL1coy0joBxGFRe/P6hYtBJIhe2ZRiap7//PCuXrAPA6XTg8rj45i8XUzZhZNT9F06PLqkba+pFplti0/73zO3vOghotG8h5N2clZuP0o1s0Re29ckbq1n9znpCgRChQAh/a4DmhhZ++Z3/JkYxxpgysfVcKujgGmj+ldmAmSDmbsml0PyY1aEJZEQubGzpCx8R8Ed3ji8fu5fgwW/hdh8G1wRU7jUo14S458mE5hYp1/o8XWuzO4Qg8Dd03o0oR54VUYl2ksiFbYWD0XVU5p15hGu+eRC3S5s9FkKr0Y1VrFtzDVvWKEZPGsUJ586KWdo2Y1rPpYJRG/u4coLRCJLILSWJXNjW/EVz2LfjQGf5XAVcvrgOj7fnvIo2gvjUc7z3h9F4fR7+8tib3Pm72ygeHt29R0bncbinQuBvmJ+O3Slwpq5AmugfmSMXtnXqFSdRNnEU3hyzgcKQoZCTF92ZXjmgfIK57DDgD9JU38xzP37pqOeWufNecq4ye5DSrTSu8kLuNXKxMw3IiFzYltvj4o4nbmHdXzfy6UfbKBmRh8u9i+i5XGio63qrG4ZB1d8293l+GZ13Ua7R6CE/gtZnIbwJHCWQcznKe7LVoQkkkQubczgdzD5zBrPPNOexjZa9Zu/Lbsk86Fe8/kxpj8cpZ/+bLvSYO9/dzOHp2Tl3rlwVUHin1WGIGCSRi4yicr+MxgD/mwAE2gxeerKYVR/m97jfnLOOLQl31T+v71HDRbb8i3QgiVxkFKWcqLzr0bnXEgk18N0rH6K1qWeNagWUTYzeMNQfvWu4VE/pueU/370dY/Uhaj6sZ/iYUs646nMMH1N6lDMKkThJ5CIjKeVh96Y2DMNJ79ZfGlj3waec95UzBnz+jumW6q1Nncd2NDp596/LmDh5Ny6jmc1/3MbfX/qE2x66hglTVkLgQ3O5nvccyLlYLhKKpElKIldKLQJ+gdkF9gmt9Y+ScV4hEuH2utFG7C2e3pzEk2jH6LzDzvs/Iff9OjbvHE7O8cWMOesAnu0t5Kn7oS1M5wdK6/9CaB268HvZ2SBZJF3Cyw+VUk7gUeB8YBpwtVJqWqLnFSJR5ZPLyC/KjTru9XlYeNlJSX++DR9uQje2kL/yAJElmo07xpBzdi678nPp+a0gBOEtEN4a71RCHJNkrCNfAGzTWm/XWgeBPwCZ3VdJ2IJScMuDX6WgKB9frg+vz4Pb4+LkS+Z1rnJJJm+et/PPnt2HKfyggYYtQ/lgxyRer6vgQLjrdnTETOZCJEEyplZGA91bVNcAJ/a+k1JqMbAYoGyUXNnPRoHWAB+9upqtqz5jWEUpCy8/kZIRRSl9zrIJI/nB6/ewcdlWWhpbmDhnHKWjU9PH84yrPsdLj7xBsL3+i2puY+LBRoZV57Ls4HgapvmYMvQQc/PqQbnAMTQlcYjsk4xEHmuSL2piUmv9OPA4wMzps/pZm05kiqbDzTzwlYdpbmgl6A/icrl479ml3PbIjUw4fmxKn9vpcjLz1KkpfQ6A0688mZrN+/jkzdW43C50xGDv7ilcPnUFlfsb+GTtKKpHD2Vz6SGmDPMzt2ReymMS2SEZibwG6L6YthzYm4Tzigzy2q/f4UhdE+FwBIBwOEw4DL/73nP8+4t3kgnX/BwOB//4b1dw4eJzqNm8l+JRRVRMLkOHd1Dp+DmVI7fxt20VVB08jqZJQzjIyqzcJSqSLxmJ/BNgklJqHLAH+BJwTRLOKzLI2verOpN4d421jTTWNlIUo4CVXZWMLKJkZNeUkXKNQxU/jDYOsXC+kzF7DT7esV8qLIqkSTiRa63DSqnbgDcxlx8+qbWuSjgykVE8vuiysQDaALfPTaA1wKtPvMMnr68GNPM+P5sLF5+LL9cb83F2pNrnxCvLkRouIqmSso5ca/0a8FoyziUy02lXnMjLj73ZWXIWzNZsE2aPJSc/hweue5h9nx0gHDJrjH/w3DI2f7yNu57+ZxwOB5s/2cZrT7xD3Z56xk6v4MLF58Rs52YnUv9cJIvs7BSD4oyrTmHHhmrWfVCF0+UEDUUjhnD9/V9i0/It1O6q60ziAOFQmLqaeqr+toWgP8Dvv/9854dAw8FGNv59M9/+zS1UTC6z6q+UFOlUYVFrDZEawA/OcSgl6cEu5P+UGBQOp4Mbf3ANB3bVsvvTPRSPKGLC7LEoBbs37e1cstedvy3A7k9r+PCF5T1G8lprAv4gLz/yBrc+dAMAwUCI1sZWCksLcDjsV2Y/3ui8PK+48z6pTOw6vA+afgCROrOAOw50/jdQ3gUpe06RPJLIxaAaUTmMEZU9O8oMLSvG4/Pgbwv0OO7L8VJQnEfrkTZi2bF+N+FQmOd/9grL//IJKIXH6+YL/3w+006egsfrJm9I9M7OdNV7dF5T56eoqHtRrveYOmJY0qdetDbgyHfBOAzorsXDTT9HO3+Octn7W082kEQuErZ9/W7++LOXqd60l5yCHM6+diHnfuUMHI7+rSmcfdYM/vjzvxDwB82v94BC4fK6mLdoDn/6xasxH1c4rIDnfvwSy19d1TktEwqEeOb+P+FymW/tcbPGcMP/v4YhpQVJ+JsOjs6CXLVN0K1VZpVhpGbqJbQedBvR2z8i4H8L8r+anOcRKWO/76Airezdtp+Hvv5rdlZVE4lEaG5o5vXfvMufHnyl3+dwe1zc8eTXGTdjDC6XE5fLSeWMCu74zdfJLfBxyqXz8Xh7Frny+jwcf+o0lv754x5z6x3Mdephtq/dyS9u+RXaZlvQKstLWDi9ssdPylrPGUfi3BABfSg5zyFSSkbkIiGvP/kOoUDPMrFBf5ClLyznopvOJSff16/zDCsfyh1Pfp3WJj8AuQVdj7v8WxcRDkVY/upKXE4nGjj3utN467cf9HneSMSg4cARtq/dyYTZY/v990pHKbsw6p4KOvrDEOUF99yBn1cMGknkIiHVm/d1Tod053S7qNtbf8yrSron8M5zuZxcc89lfPH2C2g61EzxyCLWvLshZm2IeOr3NzDhmCJJX51TLzUjWLb7QMKt55RzGNp3HgSWgO64TuEB50jwLkxe4CJlJJGLhJRNGEnt7jp0r/nVSCjM0JHFcR41MDl5PnLyzETf1uzH6Od8SSQSoXJaeVJjsVrH6LyipiCq9dyARud5N4B7GrS9BvjBsxB8n5fmFzYhiVwk5Pwbz2Ljss09lg96fG4WXDCX3MKclD3v1AUTMWJs+QdwKNWZ5D0+D7NOm5ax7dZ6t54b6OhcKQXek80fYTuSyEVCKqaU8fVfXM/zP3mZPdv24cvzccZVp3DR4nNS+rzDKkpxuV1EItHrz3OH5OLyuPD6PJz2Dydz+pWZn5w616EnY3QubEcSuUjY5LnjuffZb2IYut9LDhMVCoYIBWNcoMOse/7jt/9tUOJIJ/FG5+V5xZLQM5wkcpE0g5XEAdweN748L61N0ZuFikckd27ebrqPzjflVTN1Qq3UcMlwkshFWgj6Q6x4ay3Vm2oYNW4k8y+Y3XlhMxalYNGNZ/HKL9+Omp+/6OZzByPktHa0pYrdyUg9M0giz2JGxODt33/AB88tw98a4LgTJ/HF2y+gtGxgrdB2baxh6Ysf0XqkjdlnzWDu2TPNAll9aKxr4oHrHqbtSBsBfxCvz8Mrj7/FHU9+neEV8S9Snn3NaWgD3nzyXQJtAXILc7n0tvOZd97xA4o/E/Wu4VJd2X3Lf7VMvWQIFWsNcKrNnD5Lv/Bs/3f+idR46rt/YO37GzoLUjmUIqcgh+8+/x0KS/LZu20/bzz5LjVb9lI2aRSLrj+L8smjYp7rg+f+zosPv0YoEEZrjdfnoeK40dz+X1/rM5k/ed+zrF6yjkjE6DymlGLyCeO5/bHFff49DEMTCoTw+DwZ0WkoVZZW7erx+75QC/V51Uyd0JqSGi4i+fK8x6/UWkf1CJQReZaq39fAmnfX97hgaGhN0B/kr39cxrSTJvPQrU8QCoTQWnNgdx0blm7iGw/fGLVDsvVIGy/84tUe5wr4g1R/uodVS9Yzf9Hso8ay/oONPZI4mBUOt67aQSQc6fODwOFQeHNiN64QXRZOr4w6lg7lc0XipNZKltqzbT8uT/TneCgYZvvaXTz/078Q7FbESrcn+ed++lLUY7at3oHTHX2ugD/IqiXr+ozF4Yr9NlRKoQbxAmo2SmkNFzFoJJFnqdLRxURC0RtqnC4no8aPoHrznpiPq9m8L6oAlTfXE104D7OCYU5B35uCFpw/B1evDwKny8nM06bZsra43VSWl3Dl7GlMD05lU9VoVmyr5vXq5Rz0x34PiPQj/0qy1KjxI6icVhGVQF0uJ2d+6RRy4yRgX743ah560tzxuGOM7t0+Fwu/2HdjgktvO5/RE0fh9Xlwe934cr2Uji7h6ru/0P+/UJqIhCN8tmYn29fvxjCMvh+QRmR0bl8yR57Fbn7wOp794Z9Z8856tNYMH1PKtfddTunoEs64+hTe+u37UTsVK1QAAA2OSURBVEv7zrw6uoiSw+ngtkdu5OHbftNZUjYSCnPRzecxflb0vGxvvlwvd/7uNrat3sHebfsZNqaUqQsmHnVdeigY5tDewxQOLYhZaMsKG5dv5cl7nsEwNGiN2+vmpp9dx/iZY6wOrd/SqfWc6D9ZtSIIBcOEQ+Ee67YNw+D5n77M31/6BKfbRTgY5uSL53HVnZficMb+ImdEDDav+Ax/S4DJJ4xPWXeed59dyiuPvQVAOBxh7jkzufa+K2J+KxgsjXVNfO+LP45qWefL8/HD1+7Bm+u1KLKB21VTz8d1+2VlSxqRVSsiLrfHFZUEHQ4HV935BS6+ZRGH9tYzdFRxn0WwHE4Hx504KZWhsvrdDbz8X2/2SJir39mAw+nkK9+7MqXPfTSfvLkGHWMqRRuaNe9XceIF9qvrHav1nNRwSU8yRy6OKrfAR8WUspRWMjwWbzz1btSoNxQMseKtNQRaA3EelXoth1ti1n6JhCO0NLRaEFHyLJxeyU0zF1BZW8my5SUyd56GJJELW2msjd2WzKEULTHqrgyWKQsm4s2Jnj5xOBVT5k+0IKLk67gYenDdFJZUGZ0rW7r/CGvI1IqwlQnHj2XNexuiuhJ5fG6KhhVaFBVMmT+RibPHsm31DgLt3xi8Pg9zzp7J6EkjLYsr2XpXWKye0vPDs6JYpl6sIIlc2MrFt3yeT5dvIdgW7NE84rJvXWTpmnOl4JYHv8rHr69m+asrcbqcnHLpAmaflZkXBzvbzW1t6nF8mWEk3HpOHDtZtSJs5+DuOl799RK2r91Jyahizr/xbKYuyIzpC7vrWOkSLKtj5rhWGZ0nmaxaERlj+JhSrv+PL1kdhoghWa3nxLGRRC6ESDppPTe4JJELIVIi1ui8uKBaEnoKSCIXQqRU79ZzhyfIdEuySSIXQqSc1HBJLUnkQohB07v1nIzOk0MSuRBiUMnoPPkSSuRKqZ8AFwNB4DPgeq11QzICE0Jktnij8/K84h73k+Tet0RH5G8Dd2utw0qpB4C7gX9NPCwhRDaIVWGxqKhr23++q1qmXvohoUSutX6r26/LgSsSC0eI7NBY18Syl1dQv7+eSXMnMOfsGVHdmrJJ55b/2iao7TpeZRgy9dIPyXzn3AD8bxLPJ0RMWsOODbvZuX43haUFzDp9Gh6v2+qw+u2ztTt59BtPEg5HCIfCfPLGWt548l3+5be34rNhA4pk6RiddycXRvunz0SulFoCxCrfdq/W+qX2+9wLhIFnjnKexcBigLJR8qkqBiYSjvDLb/8321ZvJxI2cLmdPPeTl/jWr25i1PgRVofXJ63hyXufxd/WVTs90Bagds8h3v79X7n4pnMtjC79yIXR/umzXJzW+hyt9YwYPx1J/DrgIuBafZQKXFrrx7XW87TW80qKS+LdTYij+usfl7F19WcE/EHC4TD+tgAtDS38+q6nrQ6tX+r2HKKlsSXqeDgYZuWbayyIyB6kMfTRJbpqZRHmxc3Ttdb2boMibOFvf/6EoD/U45gGDu09TN3eekrL0nuQ4PK4zObMcW4T8XUfne9b1yI1XLpJ9J3zCOAF3lZKASzXWt+ccFRCxGHE6IsJZj1wIxL7tnRSPHwIo8aNoGbL3h7NMTw+D6dedqKFkdnHwumVAFJhsZuEKvFrrSdqrSu01rPbfySJi5RacP7cqEbRAIXF+QwrL7UgomP3tQe+zJDSQny5Prw+Dx6vm+mfm8KpV5xkdWi2Eq/1XDaS73LCVs66ZiHrPqhi//aD+NsCeLxuHE4nN/zwWswvhemvdHQJ9//lLjYu30LjwSOMmzmGsomZ0w5uMEn9c5N0CBK2YxgGG5ZuZvvanRSNGML8z88mb0iu1WEJi2VDdyLpECQyhsPhYNZpxzHrtOOsDkWkkajG0GPWUVFm1j/vLtOSO0giF0JkmN41XKorM3/LvyRyIRJgGJr6fYfx5XnJL8qzOhzRrvtSRbZ2Hd8Xys3IZYuSyIUYoA1LN/H0/X/E3+LHiGgmzR3H9fdfLQk9jXQsVewu1tSL3RO6JHKRkH3bD/DSo2+wfe0uCkry+fxXz2T++XNss4JkoPZ+tp8n7n66x+akLSu381+3P8Wdv7vNwshEXzKxuYUkcjFgB3fX8ZPrHyXQGkSjaW5s4dkfvsCh/Yc5/4azrA4vpd59dinhYKTHsUg4wt7tB9j72X7KJshywnSWaTVcEtoQJLLba0+8Q9AfQtO1hDXgD/LmU+9FbaPPNIf2HI65y9TpctBwoNGCiMRAZEoNF0nkYsB2bNgdM5k5HA7q9hyyIKLBM3n++Jg7TEOBMOVTyiyISAxUZXkJV86exvTgVDZVjWbFtmrb7RKVRC4GbFh57AJV4XCYIaWFgxzN4DrtipPJKcjB6XJ2HvP6PJx6+UkUDi2wMDIxUHYencscuRiwRdefzbbVO3pMo7i9bmafOSPjd1rmFeZyzzO38/pv3mH9h5+SW5DDWdecyokXnmB1aCIBdp07ly36IiGr3lnPcz95idamNhSw4Pw5/MOdX4g57SCEnXRs+a93NjN1+h6mjhhm+coW2aIvUmLu2TOZfeYMmhta8OV5bdVyTYij6Rid76qp5+N1+T02EnWXDiN1SeQiYQ6HorAk3+owhEiJqBouU3pu+S8usH5TkSRyIYToh46NRNVbmzqP7QvlsinP+k1FksiFEKKfOkbn3e2qGcnH6/ZbWsNFErkQQiQgHZpbSCIXac8wNNWb9hAOhamcVo7LLW9bkX46a7j0ujA6GKNz+Rch0lr15r089q3f4m/2g0OhFFz/H1czY+FUq0MTIopVo3PZ2SnSVigY5qFbHqehthF/WwB/i5+2Zj9P3PU09fsarA5PiLh6N4b+n43vpXSXqCRykbY2LN1EJBK9Yc2IaJa/ssKCiITov44aLpW1lSmv4SJTKyJttTS0YBiRqOPhcJgj9c0WRCTEsRuM+ueSyEXamnTCeGJVkPDleJl28uTBD0iIAUp1DReZWhFpa0TlME68cC5en6fzmNfnoXxqmVzsFLaUqgqLMiIXae3quy7juBMns/SF5YQCYeafP4eTLz4Bh0PGIMKeUjE6l0Qu0ppSMOesGcw5y779FIWIJZlz55LIhRDCIvFG592NyM3vM7lLIhdCCIt1FuSqGcHmHS2dx+udzZSP6Zp6iUcSuRBCpIHYBbnqe0y9xCOJXAgh0lT3qZd961ri3k8SuRBCpLmF0ysBuDfO7bKGSwghbE4SuRBC2JwkciGEsLmkJHKl1B1KKa2UKk3G+YQQQvRfwolcKVUBnAvsTjwcIYQQxyoZI/IHgTuBGHXqhBBCpFpCiVwpdQmwR2u9th/3XayUWqGUWlF/uD6RpxVCCNFNn+vIlVJLgJExbroXuAc4rz9PpLV+HHgcYOb0WTJ6F0KIJOkzkWutz4l1XCk1ExgHrFVKAZQDq5RSC7TW+5MapRBCiLgGvLNTa70eGN7xu1JqJzBPa12XhLiEEEL0k6wjF0IIm0tarRWt9dhknUsIIUT/yYhcCCFsThK5EELYnCRyIYSwOUnkQghhc5LIhRDC5iSRCyGEzUkiF0IIm5NELoQQNieJXAghbE4SuRBC2JwkciGEsDlJ5EIIYXOSyIUQwuYkkQshhM1JIhdCCJuTRC6EEDantB78PshKqVpg16A/cU+lgLSlM8lr0UVeiy7yWnRJl9eiUms9rPdBSxJ5OlBKrdBaz7M6jnQgr0UXeS26yGvRJd1fC5laEUIIm5NELoQQNpfNifxxqwNII/JadJHXoou8Fl3S+rXI2jlyIYTIFNk8IhdCiIwgiVwIIWxOEjmglLpDKaWVUqVWx2IVpdRPlFKblFLrlFIvKqWKrI5psCmlFimlNiultiml7rI6HqsopSqUUu8ppT5VSlUppW63OiarKaWcSqnVSqlXrI4llqxP5EqpCuBcYLfVsVjsbWCG1noWsAW42+J4BpVSygk8CpwPTAOuVkpNszYqy4SB72itjwNOAm7N4teiw+3Ap1YHEU/WJ3LgQeBOIKuv+mqt39Jah9t/XQ6UWxmPBRYA27TW27XWQeAPwKUWx2QJrfU+rfWq9j83YSaw0dZGZR2lVDlwIfCE1bHEk9WJXCl1CbBHa73W6ljSzA3A61YHMchGA9Xdfq8hi5NXB6XUWGAO8JG1kVjqPzEHe4bVgcTjsjqAVFNKLQFGxrjpXuAe4LzBjcg6R3sttNYvtd/nXsyv1s8MZmxpQMU4ltXf0pRS+cCfgG9qrY9YHY8VlFIXAQe11iuVUmdYHU88GZ/ItdbnxDqulJoJjAPWKqXAnEpYpZRaoLXeP4ghDpp4r0UHpdR1wEXA2Tr7NhjUABXdfi8H9loUi+WUUm7MJP6M1voFq+Ox0CnAJUqpCwAfUKiUelpr/WWL4+pBNgS1U0rtBOZprdOhwtmgU0otAn4OnK61rrU6nsGmlHJhXuQ9G9gDfAJco7WusjQwCyhzZPM7oF5r/U2r40kX7SPyO7TWF1kdS29ZPUcuengEKADeVkqtUUr90uqABlP7hd7bgDcxL+49l41JvN0pwD8CZ7W/F9a0j0hFmpIRuRBC2JyMyIUQwuYkkQshhM1JIhdCCJuTRC6EEDYniVwIIWxOErkQQticJHIhhLC5/wO31xhfNxVixAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x1, x2 = np.meshgrid(np.linspace(-5, 5, 100), np.linspace(-5, 5, 100))\n",
    "x_test = np.array([x1, x2]).reshape(2, -1).T\n",
    "\n",
    "feature = PolynomialFeature(1)\n",
    "X_train = feature.transform(x_train)\n",
    "X_test = feature.transform(x_test)\n",
    "\n",
    "model = LeastSquaresClassifier()\n",
    "model.fit(X_train, y_train)\n",
    "y = model.classify(X_test)\n",
    "\n",
    "plt.scatter(x_train[:, 0], x_train[:, 1], c=y_train)\n",
    "plt.contourf(x1, x2, y.reshape(100, 100), alpha=0.2)#, levels=np.linspace(0, 1, 3))\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FishersLinearDiscriminant(object):\n",
    "    \"\"\" Fisher's Linear Discriminant Model\n",
    "    \"\"\"\n",
    "    def __init__(self, w=None, threshold=None):\n",
    "        self.w = w\n",
    "        self.threshold = threshold\n",
    "\n",
    "    def fit(self, X, t):\n",
    "        \"\"\" estimate parameter given training dataset\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        X: ndarray with shape (N, D)\n",
    "           training dataset independent variable\n",
    "        t: ndarray with shape (N,)\n",
    "           training dataset dependent variable\n",
    "           binary 0 or 1\n",
    "        \"\"\"\n",
    "        X0 = X[t == 0]\n",
    "        X1 = X[t == 1]\n",
    "        m0 = np.mean(X0, axis=0)\n",
    "        m1 = np.mean(X1, axis=0)\n",
    "        cov_inclass = np.cov(X0, rowvar=False) + np.cov(X1, rowvar=False)\n",
    "        self.w = np.linalg.solve(cov_inclass, m1 - m0)\n",
    "        self.w /= np.linalg.norm(self.w).clip(min=1e-10)\n",
    "\n",
    "        m = np.mean(X, axis=0)\n",
    "        self.threshold = self.w @ m\n",
    "\n",
    "        \n",
    "    def transform(self, X):\n",
    "        \"\"\"project data\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        X: ndarray with shape (N, D)\n",
    "           \n",
    "        Returns\n",
    "        =======\n",
    "        y: ndarray with shape (N,)\n",
    "           projected data\n",
    "        \"\"\"\n",
    "        return X @ self.w\n",
    "\n",
    "    def classify(self, X):\n",
    "        \"\"\"classify input data\n",
    "        \n",
    "        Params\n",
    "        ======\n",
    "        X: ndarray with shape (N, D)\n",
    "           independent variable to be classified\n",
    "\n",
    "        Returns\n",
    "        =======\n",
    "        output: ndarray with shape (N, D)\n",
    "           binary class for each input\n",
    "        \"\"\"\n",
    "        return (X @ self.w > self.threshold).astype(np.int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAD4CAYAAAAn+OBPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5yU1b3H8c9v5pmylQXpRUCKgEtRERAxIvZIlGuJGk0sV73GmGiMseammXtNokZTNBbUeNWosRuNQRAVFRsiiAgKKiBd+rbp5/7x7LJlZndndvrM7/16+Yo75Znfmv3OOc95znOOGGNQShUPR7YLUEplloZeqSKjoVeqyGjolSoyGnqlioyVjQ/t3qOHGTBgEABBE8TtzkoZShWsjz9cts0Y0yvWc1lJ24ABg3jy2TkAbAptAGDIwH2yUYpSBWm/8qFr23su6937ftYAANas357lSpQqDlkPPWjwlcqknAg9aPCVypScCT1o8JXKhJwKPWjwlUq3nAs9aPCVSqecDD1o8JVKl5wNPWjwlUqHnA49aPCVSrWcDz1o8JVKpbwIPWjwlUqVvAk9aPCVSoW8Cj1o8JVKVt6FHjT4SiUjL0MPGnyluipvQw8afKW6Iq9DDxp8pRKVstCLiFNEPhSRF1J1zHhp8JWKXypb+suBFSk8XkI0+ErFJyWhF5GBwInA7FQcr6s0+Ep1LlUt/e3A1UCkvReIyMUiskhEFu3ckb5QavCV6ljSoReRmcBWY8wHHb3OGHOPMWaiMWZi9x7pXfm2KfhKqWipaOkPA04SkTXAY8AMEXk4BcdVSqVB0qE3xlxnjBlojBkCnAnMN8ack3RlKaBdfKWi5f11+vboub1SsaU09MaY14wxM1N5zGRo8JWKVrAtfRMNvlKtFXzoQYOvVEtFEXrQ4CvVpGhCDxp8paDIQg8afKWKLvSgwVfFrShDDxp8VbyKNvSgwVfFqahDDxp8VXyKPvSgwVfFRUPfSIOvioWGvgUNvioGGvo2NPiq0GnoY9Dgq0KmoW+HBl8VKg19BzT4qhBp6DuhwVeFRkMfBw2+KiQa+jhp8FWh0NAnQIOvCoGGPkEafJXvNPRdoMFX+UxD30UafJWvNPRJ0OCrfKShT5IGvwPhDZjav2B2XYWpvRPCG7NdkUJDnxIa/BiCqzC7rgL/axD+Avzz7Z9Dq7NdWdHT0KeIBr81U3cv4AcijY9EAB+mbnb2ilKAhj6lmoJf9IyBcDsturb0WaehV6knApS082R7j6tM0dCngXbxAe/xgLvNg27wnpCNalQLGvoU03N7m5SeBe5DARdQav+v+zCk9IwsV6asbBdQiPpZA9gU2sCa9dsZMnCfbJeTHWIhFZdD+HsQ2QyOfuCsynZVCm3p0yavW/zwdvC/DcFP7EG5ZDi7g2u0Bj6HaEufRnnX4huDqf8b+F5i75+GoxKp/BU4+2SzMpVCSbf0IjJIRF4VkRUislxELk9FYYUir1r8wNvgexkIAT77n8jXmJqbslyYSqVUdO9DwE+MMaOBKcAPRGRMCo5bMPIl+Mb3IvaEmlaPQngzhDdloySVBkmH3hizyRizuPHfa4AVgM5SaSMvgm8a2nnC0cFz8R47hGmYi9l9LWb3deB7BUyk8/eplEvpQJ6IDAEOBN5N5XELRc4Hf+8ltrac4Ny368c1BrPnJqi/H0KfQehTTN1sTM3vkh8oVAlLWehFpBx4CrjCGLMnxvMXi8giEVm0c0eO/tFnQC4HX7wzwdEL8DQ+4gDcSPllIEmM+QaXQ2gFrU8d/BD8CEKrun5c1SUpCb2IuLAD/4gx5ulYrzHG3GOMmWiMmdi9Rx6MZKdRzgbfUYJU3QqlF4BrEniPR6puBs/kpA5rgsuxBwbbCkHw46SOrRKX9CU7ERHgPmCFMeYPyZdUHHL2cp54kJJjoOSY1B3S2Q2DGwi0ecYFjm4p+xwVn1S09IcB3wVmiMiSxn++mYLjFrycbfFTzX0Ysf/UBNxTM11N0Uu6pTfGvAlICmopSjnb4qeSowKp/G9Mzc1gGrv5UoJUXAsOvesu03RGXg4oiuC7RiPd74PwGvtn55DGW3ATEN7Z+N7uqays6Gjoc0RBBj+8GUJfgdUXnIPskFtDEz9OaB2m5laINE4Qcg5CKq4Ep04H6QoNfQ4pmOCbIKbmdgguwv4TC4M1Eqm4LvHufKQBs/sGoK75sfCXmF3XIz3uBWl7z77qjN5ll2MKYXDP1D/ZGPgg0AAEIPQJpvaOxA8WeBN7pndbQQjoHLCu0NDnoLwPvm8OduBbikBwIab+XwkdyoS3EX0/AEAAwtu6WGBx09DnqPwOfqyJOI0a/maf68dJrBGAN8YzbrCGJ1qYQkOf01IZ/K9WbuDeqx/ixtNv5cFfPM6WtV8nfcx2ucZ1/Lz/7fiP5T4InP1pfU+AC6wh4KruQnFKB/JyXCoG91a+u4q7fvIgQX8IjGHLum0smb+cn8y+hIH790/sYOFdmIZHIfA+iAc8xyMlM0Gce18iZedjdn1E9Aw8gAjGhOKf2CEOpPI3mIZn7I0zRMAzAyn5j8Qv+SlAW/q8kGyL/9jvniXoC+69o82EIwQa/Dx12wuJHSjSgNl9Ffjng9kFkS3Q8Cim9vbWr3P2g8obif3n5QT3JAh/Za+Bb2IN0rXh8CJlZyE97ka634WUfhsk1t2AKh7a0ueJrrb4QX+QbRt2xHzuy4/XJVSD8b8CphYIt3g0AIH37H3qnM29BnH2wXhPBd9z2KPvBnCBZwbU3oqJfI39peCA8ssQ1zgQr7beGaChzyNdCb7TcmK5LILh6K52WbeyxAoIfkLsLrsTQp/boQ99ian9Y/Nmlc7BYI2yA+2eDDW/B9Nm1L32dxgcgBdKTkVKZmn400i793km0a6+w+lg2qxJuLytu8Mur5ujzzk8sQ939qfddsLRGyI1mN0/g/A67NY9BOEvIfAeUnoWYhoaewqxRIB6aPgHpuG5xOpSCdHQ56FEgz/r8hM46OhxWB4Lb7kHy2PxjdOmcOSZhyX0ueI9DnC2edQJzl5gjcT4XqN11x8gAqYGgkvscYBO+aHhKV1RJ420e5+nEunqWy6L7/3y25xyxYns2LST3oP2wVvehbvbnL2Qyp9jav8MkcYuulWNlF9ud8cjG2lvxJ7IVnBNJPbsurbqG1+ng3XpoKHPY4me45dXlVFeleB5fFuu0UjVHRDZjTENiNlJU5DFNQrjf53oyTkC1jBw9rT3uPPNJfYsu6aXV9Hun6YJ23vd+18BDLhnIN6jklvOq8jof6k8l9KbdCI19vV3AuA6CJy923mhsa/V+19tXBEnaL++/AfgqGrsBTS16G6wRoI1AgApPR+s0RjfvyCy3e4B0HJVXDeUfi/2QJ4x9mKawWXs/dIIrcME3kUq/1sH/+KkoS8AKQm+fxGm9hbsYZ4I8ACUnIaUnh71UtPwHPhfZ+9gHUBwMdQ/gnT7Hab+cQgsBCzwHIWUntL8ZhHwHIp4DrV/DnyEqX8Iwhvs04eSs8AzJXaNoU9bB94uHEIr7cU33Z3M0Is0gNlpL/4ZXoPxzQfjQ9xTwT2xaL40NPQFoin4XRJpwNTeStT5eMPT4D4weo6774Xo1xK0J+2UXYSUXwhcGN9nu8ch7pvje21wOdE38gD4INRB6E0QU3dP4xeVRfO8gQhgMIF3wTXWXsmnCIKvo/cKgh8Qe8WzQOOIfBumvp0DtWj508HRjeg97wE89mlFO0zdA+B/g+btukLYVxmarhD47B5E4IMUF5ybNPQFpmtTddteZmtiiBlia3TslzsGpHdRC/dhxP5yksbnYjBB8L9C7KsKLfkwwQRuBMpjGvoC0uU5+q4DiR18DxIjTFJ+Pvbtrk3X7MV+bfkliX1uohwlSOUvQbo3fr4XpAqp/Dk4ymO/xzTQ3KJ3REBKU1ZqLtNz+gLTpUE9RyWUXQh1s7HPc8OABzzTYt++6hyEVN1uD+iFVoE1yJ466xyUwt+kHa4RSPfZ8S+wKRUgZWB2d3JgJ7iPSFGRuU1MFmY+VY8db558dk7GP7eYNA3qJTSaH96E8S0A/Ih7Elj7F8TAlvG/CbV/IbqL33SlAsACKUe6/SozX15ptl/50A+MMRNjPafd+wLVpa6+sx9SdgZS9j1wjbIH7Pzv2ANcJtaoeQzhzfYiGaFVOTOVVjzTkIobwDoApCe4DoXS/6T1n38IzC7M7v/JmbrTRbv3BSyZ6/fGNx/q7qb5T0SQyuvBNaadN0QwtX+CwNvsPdd3dAephPDnQAmUnIiUnNpqwY2McY9F3GOby939G2IOUpo99k1C1n6Zqy3DtKUvcF1q8cProe4emlezbQDqMXt+A5HY698Z34sQeKfxPT77n8gmCH+KHa4aaHgaU/fXJH6bVGpo53Fp3oWnQGnoi0CiwTe+V2n3enuwnWvZvpfo/LJYAPwLmneqySb3YcS+5m/2ThkuVBr6ItHPGkAkHOGVFz7g3RcWs31jB8Ez9bSeD9/yuXZayLhbR3fzTjVZJN6jwTkQ8DQ+4gDcUH5pwS/Fpef0RWLTF1v48yV/IzIgCCYCNQ0cfsoUTrtyZtQIvbgntXO3XBhcE2J/gPsQexpuuxN9mgTB0aeLv0UKiRvpdhME3sIEFoGjCvEcB1b+j9x3Rlv6YmDgjssfoGZnLXUf+wn4gwTcFm89+x5LX/8k+vWuCeAaS+v15t1Qcop9e2zU8Q14TwAqaO4yxxqsc4PrEHDmyJZd4gLPdKTiKqTswqIIPGhLXxS++nQD9bub58sHV4NrOPgsB288+Q7jpx/Q+g0i9s0ngfcwgTdBPIjnKHBFT781/nfsQT9TB5jGyTKVdoCskdDwpD0ajge8xyKl56T1d1Wd09AXgYA/iDhad+Gbgr8r0M6AnQh4JmOsQ5j70OvM//uT+GobGDp2MKdd+S17vfzgSqi9nVYDeOG14JqIlJ1r/+w51D6dQApiok8h0O59ERg8emDsJ9a7GDN1ZIej+v/4/XO8NHs+tTtqCQXCrPrgC2696C62rtuGqX+KmLfYBt+HcIv18MSR2cCbMPgXge9lCK3J3OfmCQ19EbDcFt/9xem4PC4cln2u7S7xMGBYX46bcQwQ+3Je7a463n5hEUFf62AH/UFefvA1e7OLmFxgYq+1n3bhzZidF2Nq/4Cpux+z+1rMnt/ZXwQK0O59Qvz1fub//Q0WzVmKy+Pi8NMOZepJE6O6zrlowpHVXP/3K1j43Pvs3l5D9dT9mTCjGqflpB+xZ+59vW4blssi5G99CmDCEdatWG9P1fVvJPryXhgc/dL/S8Vgam5uXHW3xVTa4BKMbw5S8s2s1JRrUhJ6ETke+CP2kO1sY8xvU3HcXBIKhLj5gjvZ9tU2go3nwU/c+jyrPvic8248M8vVxaf3vj2Z9cMTYj4Xa8ruPgN6EIpxzi8Oof+wvkjJdIz/LVrPbvNAySxwdGG13WSFt9uzCaNupfWDfw5o6IEUdO9FxAncAZwAjAHOEpF2Jmjnrw9f+ZjtG3fuDTxA0Bdgyasfs/nLrVmsLHXaztyr3KeCCTOqozbKsNwujj1vOjj7IlW/B9cUoAIcA6HsYqTk2xmuvEmQ2ItsEP8NQ0UgFS39JGC1MeYLABF5DDgZiHEBOH99+v5qAg3RyzaLCF98tJa+Q9tbOTa/tG3xv/uL06ncp5w3n36PgC9I/2F9OOOaWfQf1td+g3MAUnl1dotu4ugD0g1M2224LXAnuJtPAUtF6AcAX7X4eT0wue2LRORi4GKA/v0HpOBjM6t73yr7/DbYursrDgfdelVmqar0aBv8U3/8LU69YibhcASnlYU75OIlglT8GLPn19gzA4OAt3GV3VlZLi53pCL0sfpTUTckG2PuAe4BexGNFHxuRk09eSLzHnq91WKsIoK31M3oybl1g8aWtV/zxlPvsmvLLsZM3Z9Djp+Ay5PYfPKoc3yR3A58E9copOoOjP9ViGxBXNXgnpL4fPpIA6bheQi8aa/75z0e8RxdEHMNUhH69UDL+YsDgY0pOG5O6d6niu/fdh4P/Pdj+Gp9GGPoPagnF9/8XRzOxIdGTMSwZvlX+BsCDB27L56S1CwouWzBCu674RHCwTCRcITlCz/llYcX8NMHL8Nb6un8AKE1ENkJ1n6p3Ugjk5zdW6+1nygTxOy+pvGSZOO3fN39mOAnSMXlKSkxm1IR+veBESIyFNgAnAl8JwXHzTkjJw7jf1+8ni1rv8byWPTs36NLx9n4+Wbu/NED1Nc2ICKEwxHOvHYWU048OKn6IuEI//fLxwn6mrsjAV+A7Rt38Npjb3H8BTM6ePNuzJ4bIbyhcdvoEOL9Jv1Kz2VTOA+Dn4zAW4279LQc/PPbG3iETwNn/p2etpT06L0xJgRcBswBVgD/MMYsT/a4uUocQt+hvbsc+Eg4wp8uvZedW3fhr/fjq/MR9AV47LfPsGHV5qRq27h6M+Fw9C2xwUCIxfM+6vC9puYWTHgt4EdoQAgS2P1Pdm96seur7OYpE1xG9B2GAE4IfpbpclIuJTPyjDH/MsaMNMYMM8b8TyqOWag+W/Q5QV/0te9QIMybz7yT1LHdJW4iMUIP4C3roGsf2YUJfYa0uS3W7Ymwe/3f8dX74w++CULgQ3ttvUhdQvXnDOlJu53gDjbVyBc6DTfD6vfEXoTCRCLU7EwuJL337UnPgfsgbQab3F43R5w+tf03mjoi7cxSLSkNsejfS4A4VuAJrsTsuABTczOm9s+YnRdgfHMT/j2yTbxHE31rsNhLabvGZ6OklNLQZ9iwA4dGXfYDO5jjv9H6Ftc922t4afY87r/+77z66Jv4ajtfneaSW8+le58qPKUevKVeXG6LqScfwsHHdPDH6uhHOBzdsoWCsHRhGZvXNE8+ajf4JmCPCVCH3TVuAIJQdx+E1nVad05x9kIqrm/cMtsDuMA5GOl2o33zUJ7TufcZ1q1nBceedyTzHnqdQOONLC6vm/7D+nDg0c0bS6z/bCO3XXQ34VCIYCDER2+sYM6Dr3Ht//2Qqt7d2j1+zwE9+PVz17B6yZfs2VbDfuMH071PJ11ScbBt5xnsU/UglmVwOiHgF+prHCz4Z19mfr/1XXoxR/UDi4m9k0wI438Fsc6P5z9P7nCPRbrfB5ENgAecvbJdUcpo6DNo19bd1O6s47jzpjNswhDeePJtGmp9HHT0OKbMPBjL1fx/x8O/fhJffXPLHvQFCAVCPPvnlzqd6y8OYcRBiS3hPGD0Sdx/3XJGT/iM7r0DrFxcytv/7oG7pJKDjh4b9fro4Le3umykgw0vc5xI4zp6hUVDnwF1u+u595qHWbNsLU6XhYkYTvnxiVz0++/GfL2/3s+G1dGLR5pIhI/fXNHhZ0UiET5Z+Bmbv9xK3yG9GXPYSByOOLqkAmf//Eqev3MOT9/7ISYSYfz0av7jRydguWP/mbQKfr9xxF5B14u4oiZoqizS0GfAPT99iC+XrSUcCu+9YefJP7xA70E9GTlxWNTrnZYzajCuidXBzLq63fXc8p93svvrPQQDIVxui8p9Krjq/kspryrrtE5PqYfTrzqJ0686Kc7frEXwN8HgHqfbe9rTdI+CF6wx4E5u/oFKrfwflchx2zfuZO3ydYRDrYfHg74A8x5eEPM9ltui+vDRUdNeXW6LabMmtftZT9zyPNs37MBf7ycSCuOv97N9006euPn55H+RDjQN7q3dMcPeQdYzA1xTkfIf2rviFMDU1UKioU+zmh21OF2xO1S7trS/k+rZN5xKv6F98JR48JR4cHldjJg4vMNZdUvmL4v6comEwnw4/+P4dmuOg78hwPy/v8GtF/6Vu658kBXvrAJajOpv6Y2UX4ZUXmWvj1cAo92FRrv3adZvWJ+Ys+SclsWYqfu3+76ybqVc98jlfLlsLds27GTAiL70H9435msDviALnljY6l7/llK1M3HAF+Tm8+5g24btBP32FNVP31/N8RfM4Ljzj9zb1Ve5Tb+G08xT4ubkS4/D5W2+ocZpOSmt8HLU2Z3c4y0wdNxgDjlhQruBD4fC/OGiv/LC3bEnwYjDQfW0Ue2uLZGId1/4gG0bmwMP9tz+l2bPo3ZXns6+K0La0mfAkWdNo/fgXsx7aAF7ttcw5tCRHHvudCp6lCd97KWvLmfL2m2tgtjE5XVTVlHCGVefnPTnAHy04JNWN/M0cbos1iz7iurDRwEU1805eUhDnyEHTN2fAzroznfVivdWxV7Rx+lg4rHjOfOaWe1ecktUt56l9B4YpGaXg4ba5kFGEzGUdbPXxMvb23GLiIY+h9XsqGX9Z5vo3ruSvvvF3v+tqlc3LMsiFGp9Pu/2uBl/xAEpC7zxzePM7z9PyO/H4YRl75Tx0K19CPmdlHcvY+jYwXtfq8HPbRr6NDIRw9frt+Mt81C5T0UCb4Sn//Qir/9jIZbbIhwKM2B4Py69/Xw+X7qGf937Cru27GLwAYOYfsZUHJaj1bwYAdweizFTR6bmFwkshbr7cDr8OBsXuR07uY7zr93KM/dXc9mfLogaM9Dg5y5J1chuIqrHjjdPPjsn45+bSR+/tZKHf/UE/oYAkXCEwQcM4sLfnh1X+N99cTGP/faZvXPzwR786z2wJ9s279y7+YRgn7ef+uMT+edf5xL0B4mYCN17V/Fft55L3yGpmS9u9vwCgsuiHo8YC0ePexFH+/cCNI3ma/Aza7/yoR8YYybGek5b+jTY/OVWZl/7cKtBry+XreVPP5jNzx79cacj6fMfeaNV4MEepd+0pvWOMgZ7ks/Hb6zkpjk3sHH1Flweiz779krJaH3zh7ddXdbmcLiQyB7oIPTa4ucevWSXBq/9YyGhQJtJMuEI2zfsYN3K9Z2+v64m/htUDLBm+Vc4HA4GjuxHn8EpDjyA6wBi/6kYcMa+lNhSsa28k+s09GmwfcMOTCR6Qo7D6WDX13s6fX/1YaP27jkXj05vnU2SlJwO4qX1t4kHSs6Oe5VZDX7u0NCnwf6ThsdccjocDLFvezvItnDChUdTVlkS1x55Lq+bEy46qkt1xs3ZG+l2C3img/QCa3+k4kqk5MSEDqPBzw0a+jSYNmsS5VVlOK3mIRO3183UWZOoimNjjG49K/jZ41fSd0jsy3RgD+yVVJRw2pUzGfeNDOwi5uxr30DT426k203gPqRLh9HgZ58O5KWBt9zLtQ//iDl/e5WPXvuEkgovR545jcnfPCjuY5RXlTH9jKk8ddsLUYN6LrfFZX+5kP3GDe7SmvvZpoN72aWX7HJYwBfkf8+6jZ1bdu9dV8/ldXPgUWM595fZ2iQydfRyXvroJbs85fa6uPrBH/Lyg6/x4fxluL0ujjh9Kod1cE99PtEWPzu0pVdZpy1+6mlLr9Ju55ZdvPviYup21zPm0P0ZNWl4XFcfQFv8TNPQq6Q1bZppQoZQKMSbz7zH8AlD+P5t57Ftww7eevZd9uyoo3rq/kyYUR1z91sNfuZo914lJRQIcc0xN7ZarhvsS5SHnjSRhc+9TzgUIRIO4y7xMGBYH664+7/avftPu/qp0VH3Pv+u96ic8sVHa2NO+w34Aix48m37JqCwPSU50OBn/erNvP3PRe0eT6/jp5+GPgV2btnF3P97jefu+DerFn+RskUo84HTcrb/+8ZYBTfoC7BozpIOj6nBTy89p0/S0teW88DPHsWE7fPZ1x57iwOm7s9/3nR23ANZ+Wzo2H2x3E5oc4+Qy2OBgWCMRUE9pR3soNtIz/HTR1v6JAR8QR78+eME/cG9K9cEfAGWL/yUpa8tz3J1meFwOrjkD+fhLfPiKfXgclu4PC4mnziRshgbbLi9bg4/ZUpcx9YWPz20pU/C6sVfxGzNA74AL9w9l63rtjHuiDH0Hdo7C9VlztCx+/K/L93Astc/oXZPPaMmjaDvkF5sXH0of/z+vYSCISIRgwmHOfy0KYw9fHTcx9YWP/V09D4JK99dxb3XPIyvLnoLaRFBHA6cloOjzvkG37rk2CxUmH3hUJgV76yibk89Iw7cjx79unYbsI7qJ0ZH79Nk+EFD291zzhhDJBwm6A/yyiMLWP/pxgxXlxuclpPqaaOY/M2Duhx40K5+KiUVehG5WURWishHIvKMiKR3NYccY7ks/uuW7+H2unF7PTicsRe+CPlDLHp5aYarKzwa/NRItqWfC1QbY8YBnwHXJV9Sfhlx8H7c9NINfPunJzFhRnXMxTOAnNjEMRQIEWpn66t8ocFPXlIDecaYl1v8+A5wWnLl5CdvuZdDT5pI9bRRLHs9etTecltMPHZcFiqz7dq6m4dvfJJP3/8cMAwbP4Rzfn46PQf0yFpNydDBveSk8pz+AuClFB4v71T0KOfMa/8Dl9vCclk4LScuj4tjvjedgSP7d/m4NTtqef7Of3Pz+Xdw/w2PsvaTzhfXbBIOhbnlgjtZ+d5qIuEwkXCE1R9+yS3n30EgxhZV+UJb/K7rtKUXkXlArCVPbzDGPNf4mhuwt1t4pIPjXAxcDNC//4AuFZsPpnxrIvtPGs6SVz4mHAoz9ogx9gq1XbRr625uOvtP+Op8hIIh1i7/io8WLOe8X53JhBnVnb5/2YIV1Nc0tFqo0xiDvyHI4rlLmfKtmAO8eUFb/K7pNPTGmKM7el5EzgVmAkeZDq7/GWPuAe4B+5JdgnXmle59qjjyO9NScqyXZs+nvqZh7/x1YwxBX5BHb3qGcdPH4HB03Fn7ev12gv7o8/iAz8/Wr7alpMZs0uAnLtnR++OBa4CTjDHxL9au4rb87ZV7A99SwBdg+4Ydnb5/wIh+uGLc0eYp8TBwZGH0uLSrn5hkz+n/AlQAc0VkiYjclYKaVAvlMaaygr15RklFSafvHz15BD0H9sByNQffaTmp3KeC8dMzsIpuhjQFX3UuqdAbY4YbYwYZYyY0/nNJqgpTtqPP/gZur7vVY07LYuQhw9v9QmhJHMKV917C1FmTKK0spaTcy+SZB/PTv/0g5mIWqvDp3PscN/G4CWz6YgvzHlmAy20RCoYZPHog5994ZtzH8JZ5OePqkznj6pPjev22DTuo3VXHgOF92593oPKWhj7XCXzr0uOYcc7hbFy1mW69Kum9b8+0fGYm/dQAAAcnSURBVFTNjlru+smDbFi1CaflJBIxnHrFiUw7ZXJaPi8ddECvczr3Pk+UVZYy4uD90hZ4gL/++AHWrdhA0B/EV+cj0ODnydteYPXiL9P2mamkA3rx0dArALau28bGL7ZEXSkI+gLMe2RBlqpKnAa/c9q9z2P+hgALnljIB3OX4S3zMP3bU5lwZHWXtqqu2VGL03ISJHqW3u44dtrNJXrtvmMa+jwVCoS4+fw72LZ+O0G/HdS1y7/i84/WcOoVMxM+3sCR/QgHo+cDWC6LA6aNSrreTNPgt0+793lq0ZwlbN+wY2/gwZ6w88YTb7Nr6+6Ej+cp9XDSpcfjanF50HJZlHcv48gzD0tJzZmmXf3YtKXPUx+/tTJqN1uwr+GvXrKGiceOT/iYM74zjf7D+jDv4Teo2V5D9bRRzDj7cMoqS1NRclZoix9NQ5+nqnp3w+F0EGmz2qzBUNmjvMvHHTV5BKMmj0i2vJyiwW9Nu/d5atopU6Jm1AlCaUUJIw7aL0tV5S7t6jfT0OepvkN6cd6vz6Sk3Iu31Ivb66b34J5c/teLimK9/a7Q4Nu0e5/HJsyoZuw3RrP+s014Stz0HdK7S5friol29bWlz3tOy8ngMQPttfU18HEp9hZfQ6+KUjEHX0OvilaxBl9Dr4paMQZfQ5/jfHU+li1Ywcr3VhMORU+TVckrtuDr6H0Oe/v5RTz++2f3Xo93OB1cevv5DB27b5YrKzzFNKqvLX2O2vzFFh7//bN772331fmo31PPHT+6L+93qclVxdLia+hz1MJ/LorZnTcRWL7w0yxUVByKIfga+hxVv6chal49QMRE8NX5s1BR8Sj04Gvoc9S4I8bgLvFEPW7CEfY/ZFgWKiouhRx8DX2Oqp42imHjB+9d/loAt9fNsecdSVXvbtktrkgUavB19D5HORwOLv3j+Xw4bxkfzF2Ku8TNtFmTGX7Q0GyXVlQKcVRfQ5/DHA4HBx87noO7sCCGSp1CC75275WKQyF19TX0SsWpUIKvoVcqAYUQfA29UgnK9+Br6JXqgnwOvoZeqS7K1+Br6JVKQj4GX0OvVJLyLfgaeqVSIJ+Cr6FXKkWagp/rUhJ6EblKRIyI9EzF8ZRS6ZN06EVkEHAMsC75cpTKf7nexU9FS38bcDVgUnAspfJaPpzbJxV6ETkJ2GCMWRrHay8WkUUismjnjtz9D6JUsnI9+J3eWisi84C+MZ66AbgeODaeDzLG3APcA1A9drz2ClRBy+XbcTsNvTHm6FiPi8hYYCiwVEQABgKLRWSSMWZzSqtUKg/lavC73L03xiwzxvQ2xgwxxgwB1gMHaeCVapaLXX29Tq9UmuVa8FMW+sYWf1uqjqdUIcml4GtLr1SG5ErwNfRKZVAuBF9Dr1SGZTv4GnqlsiCbwdfQK5Ul2Qq+hl6pLMpG8DX0SmVZpoOvoVcqB2Qy+Bp6pXJEpoKvoVcqh2Qi+Bp6pXJMuoOvoVcqB6Uz+Bp6pXJUuoKvoVcqh6Uj+Bp6pXJcqoOvoVcqD6Qy+GJM5teoFJGvgbVpOHRPIF8W8sinWiG/6s2nWiE99Q42xvSK9URWQp8uIrLIGDMx23XEI59qhfyqN59qhczXq917pYqMhl6pIlNoob8n2wUkIJ9qhfyqN59qhQzXW1Dn9EqpzhVaS6+U6oSGXqkiU5ChF5GrRMSISM9s19IREblZRFaKyEci8oyIVGW7prZE5HgR+VREVovItdmupyMiMkhEXhWRFSKyXEQuz3ZNnRERp4h8KCIvZOozCy70IjIIOAZYl+1a4jAXqDbGjAM+A67Lcj2tiIgTuAM4ARgDnCUiY7JbVYdCwE+MMaOBKcAPcrxegMuBFZn8wIILPXAbcDWQ8yOUxpiXjTGhxh/fwd75N5dMAlYbY74wxgSAx4CTs1xTu4wxm4wxixv/vQY7TAOyW1X7RGQgcCIwO5OfW1ChF5GTgA3GmKXZrqULLgBeynYRbQwAvmrx83pyOEQticgQ4EDg3exW0qHbsRuoSCY/tNP96XONiMwD+sZ46gbgeuDYzFbUsY7qNcY81/iaG7C7po9ksrY4SIzHcr4HJSLlwFPAFcaYPdmuJxYRmQlsNcZ8ICLTM/nZeRd6Y8zRsR4XkbHAUGCpiIDdVV4sIpOMMZszWGIr7dXbRETOBWYCR5ncmzSxHhjU4ueBwMYs1RIXEXFhB/4RY8zT2a6nA4cBJ4nINwEvUCkiDxtjzkn3Bxfs5BwRWQNMzOXts0XkeOAPwBHGmK+zXU9bImJhDzAeBWwA3ge+Y4xZntXC2iH2t/2DwA5jzBXZridejS39VcaYmZn4vII6p89DfwEqgLkiskRE7sp2QS01DjJeBszBHhT7R64GvtFhwHeBGY3/PZc0tqSqhYJt6ZVSsWlLr1SR0dArVWQ09EoVGQ29UkVGQ69UkdHQK1VkNPRKFZn/Bw4h2UXhXDrbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x_train, y_train = create_toy_data()\n",
    "x1_test, x2_test = np.meshgrid(np.linspace(-5, 5, 100), np.linspace(-5, 5, 100))\n",
    "x_test = np.array([x1_test, x2_test]).reshape(2, -1).T\n",
    "\n",
    "model = FishersLinearDiscriminant()\n",
    "model.fit(x_train, y_train)\n",
    "y = model.classify(x_test)\n",
    "\n",
    "plt.scatter(x_train[:, 0], x_train[:, 1], c=y_train)\n",
    "plt.contourf(x1_test, x2_test, y.reshape(100, 100), alpha=0.2, levels=np.linspace(0, 1, 3))\n",
    "plt.xlim(-5, 5)\n",
    "plt.ylim(-5, 5)\n",
    "plt.gca().set_aspect('equal', adjustable='box')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
